internal/generator/robots.go
48 Zeilen · 1.3 KB · Go
| 1 | package generator |
| 2 | |
| 3 | import ( |
| 4 | "io" |
| 5 | "log" |
| 6 | "os" |
| 7 | "strings" |
| 8 | |
| 9 | "github.com/ruedigerp/newblog/internal/config" |
| 10 | ) |
| 11 | |
| 12 | // RenderRobotsTxt writes a robots.txt for the site. Without explicit rules the |
| 13 | // whole site is allowed; the sitemap is referenced automatically whenever |
| 14 | // sitemap generation is enabled. |
| 15 | func RenderRobotsTxt(w io.Writer, cfg *config.Config) error { |
| 16 | bw := &errWriter{w: w} |
| 17 | |
| 18 | bw.printf("User-agent: %s\n", cfg.Robots.UserAgentOrDefault()) |
| 19 | for _, rule := range cfg.Robots.Allow { |
| 20 | bw.printf("Allow: %s\n", rule) |
| 21 | } |
| 22 | for _, rule := range cfg.Robots.Disallow { |
| 23 | bw.printf("Disallow: %s\n", rule) |
| 24 | } |
| 25 | // A crawler needs at least one rule; allow everything when none are configured. |
| 26 | if len(cfg.Robots.Allow) == 0 && len(cfg.Robots.Disallow) == 0 { |
| 27 | bw.printf("Allow: /\n") |
| 28 | } |
| 29 | |
| 30 | if cfg.Sitemap.EnabledOrDefault() && cfg.BaseURL != "" { |
| 31 | bw.printf("\nSitemap: %s/sitemap.xml\n", strings.TrimRight(cfg.BaseURL, "/")) |
| 32 | } |
| 33 | |
| 34 | return bw.err |
| 35 | } |
| 36 | |
| 37 | // GenerateRobotsTxt writes robots.txt into outDir, honoring cfg.Robots.Enabled |
| 38 | // (defaults to true). |
| 39 | func GenerateRobotsTxt(cfg *config.Config, outDir string) { |
| 40 | if !cfg.Robots.EnabledOrDefault() { |
| 41 | return |
| 42 | } |
| 43 | if err := writeFile(outDir+"/robots.txt", func(f *os.File) error { |
| 44 | return RenderRobotsTxt(f, cfg) |
| 45 | }); err != nil { |
| 46 | log.Printf("Error writing robots.txt: %v", err) |
| 47 | } |
| 48 | } |