### Sitemap

Parallel zum RSS-Feed wird eine `sitemap.xml` nach dem [sitemaps.org-Protokoll 0.9](https://www.sitemaps.org/protocol.html) erzeugt - unter `/sitemap.xml` sowohl im dynamischen Server als auch in der statischen Generierung. Enthalten sind:

- Startseite und alle Pagination-Seiten (`/page/2/`, ...)

- Alle Blog-Posts (`lastmod` aus `update`, sonst `date`)

- Alle statischen Seiten aus `sites/`

- Tag- und Kategorie-Übersichten sowie alle einzelnen Tag-/Kategorie-Seiten

Die Sitemap lässt sich in der Config an- und abschalten (Default: aktiv):

```yaml

sitemap:

  enabled: true   # false schaltet /sitemap.xml komplett ab

```

### robots.txt

Ebenfalls automatisch erzeugt: eine `robots.txt` unter `/robots.txt`. Sie verweist automatisch auf die Sitemap, sobald diese aktiviert ist, und lässt sich in der Config konfigurieren (Default: aktiv):

```yaml

robots:

  enabled: true

  userAgent: "*"        # optional, Default "*"

  disallow:             # optional

    - /search-index.json

  allow:                # optional

    - /

```

Ohne eigene `allow`/`disallow`-Regeln wird `Allow: /` geschrieben, d.h. die ganze Seite ist freigegeben:

```

User-agent: *

Allow: /

Sitemap: https://blog.kuepper.nrw/sitemap.xml

```

Ist `sitemap.enabled: false` oder `baseURL` leer, entfällt die `Sitemap:`-Zeile.

│   │   ├── robots.go                # robots.txt Generator

│       ├── rss.go                   # RSS Feed Generator

│       └── sitemap.go               # sitemap.xml Generator

| `baseURL` | - | Basis-URL für RSS, Sitemap und absolute Links |

| `sitemap.enabled` | `true` | Erzeugt/serviert `/sitemap.xml` |

| `robots.enabled` | `true` | Erzeugt/serviert `/robots.txt` (mit Sitemap-Verweis) |

| `robots.userAgent` | `*` | User-agent-Zeile in der `robots.txt` |

| `robots.allow` | `[]` | Zusätzliche `Allow:`-Regeln |

| `robots.disallow` | `[]` | Zusätzliche `Disallow:`-Regeln |

| `/sitemap.xml` | Sitemap für Suchmaschinen (falls aktiviert) |

| `/robots.txt` | Crawler-Regeln + Sitemap-Verweis (falls aktiviert) |

### Sitemap

Alongside the RSS feed, a `sitemap.xml` following the [sitemaps.org protocol 0.9](https://www.sitemaps.org/protocol.html) is produced — served at `/sitemap.xml` by the dynamic server and written by the static generator. It contains:

- Homepage and every pagination page (`/page/2/`, ...)

- All blog posts (`lastmod` from `update`, falling back to `date`)

- All static pages from `sites/`

- Tag and category overviews plus every single tag/category page

The sitemap can be switched on and off in the config (default: on):

```yaml

sitemap:

  enabled: true   # false disables /sitemap.xml entirely

```

### robots.txt

A `robots.txt` is generated as well and served at `/robots.txt`. It references the sitemap automatically whenever sitemap generation is enabled, and is configurable (default: on):

```yaml

robots:

  enabled: true

  userAgent: "*"        # optional, defaults to "*"

  disallow:             # optional

    - /search-index.json

  allow:                # optional

    - /

```

Without custom `allow`/`disallow` rules an `Allow: /` is written, i.e. the whole site is crawlable:

```

User-agent: *

Allow: /

Sitemap: https://blog.kuepper.nrw/sitemap.xml

```

If `sitemap.enabled: false` or `baseURL` is empty, the `Sitemap:` line is omitted.

│   │   ├── robots.go                # robots.txt generator

│       ├── rss.go                   # RSS feed generator

│       └── sitemap.go               # sitemap.xml generator

| `baseURL` | — | Base URL for RSS, sitemap and absolute links |

| `sitemap.enabled` | `true` | Generates/serves `/sitemap.xml` |

| `robots.enabled` | `true` | Generates/serves `/robots.txt` (with sitemap reference) |

| `robots.userAgent` | `*` | User-agent line in `robots.txt` |

| `robots.allow` | `[]` | Additional `Allow:` rules |

| `robots.disallow` | `[]` | Additional `Disallow:` rules |

| `/sitemap.xml` | Sitemap for search engines (if enabled) |

| `/robots.txt` | Crawler rules + sitemap reference (if enabled) |

# sitemap.xml für Suchmaschinen (Standard: true)

sitemap:

  enabled: true

# robots.txt (Standard: true), verweist automatisch auf die Sitemap

robots:

  enabled: true

  # userAgent: "*"

  # allow:

  #   - /

  # disallow:

  #   - /search-index.json

// SitemapConfig toggles generation/serving of sitemap.xml for search engines.

// Defaults to true so behaviour without the block stays unchanged.

type SitemapConfig struct {

	Enabled *bool `yaml:"enabled,omitempty"` // sitemap.xml

}

// EnabledOrDefault returns whether sitemap.xml should be produced (default: true).

func (s SitemapConfig) EnabledOrDefault() bool {

	if s.Enabled == nil {

		return true

	}

	return *s.Enabled

}

// RobotsConfig toggles generation/serving of robots.txt and lets the site

// define its own Allow/Disallow rules. Defaults to true so behaviour without

// the block stays unchanged.

type RobotsConfig struct {

	Enabled   *bool    `yaml:"enabled,omitempty"`   // robots.txt

	UserAgent string   `yaml:"userAgent,omitempty"` // default: "*"

	Allow     []string `yaml:"allow,omitempty"`

	Disallow  []string `yaml:"disallow,omitempty"`

}

// EnabledOrDefault returns whether robots.txt should be produced (default: true).

func (r RobotsConfig) EnabledOrDefault() bool {

	if r.Enabled == nil {

		return true

	}

	return *r.Enabled

}

// UserAgentOrDefault returns the User-agent line's value (default: "*").

func (r RobotsConfig) UserAgentOrDefault() string {

	if r.UserAgent == "" {

		return "*"

	}

	return r.UserAgent

}

	Title        string        `yaml:"title"`

	Description  string        `yaml:"description"`

	Author       string        `yaml:"author"`

	BaseURL      string        `yaml:"baseURL"`

	Port         int           `yaml:"port"`

	PostsPerPage int           `yaml:"postsPerPage"`

	ContentDir   string        `yaml:"contentDir"`

	SitesDir     string        `yaml:"sitesDir"`

	StaticDir    string        `yaml:"staticDir"`

	OutputDir    string        `yaml:"outputDir"`

	ThemesDir    string        `yaml:"themesDir"`

	Theme        string        `yaml:"theme"`

	Menu         []MenuItem    `yaml:"menu"`

	LLMs         LLMsConfig    `yaml:"llms"`

	Sitemap      SitemapConfig `yaml:"sitemap"`

	Robots       RobotsConfig  `yaml:"robots"`

	// 8b. sitemap.xml for search engines

	if cfg.Sitemap.EnabledOrDefault() {

		writeTemplate(filepath.Join(out, "sitemap.xml"), func(f *os.File) error {

			return templates.RenderSitemap(f, site, posts, pages, cfg.PostsPerPage)

		})

	}

	// 8c. robots.txt (references the sitemap when enabled)

	GenerateRobotsTxt(cfg, out)

	extras := "RSS feed"

	if cfg.Sitemap.EnabledOrDefault() {

		extras += ", sitemap"

	}

	if cfg.Robots.EnabledOrDefault() {

		extras += ", robots.txt"

	}

	log.Printf("Generated %d posts, %d pages, %d tag pages, %d category pages, %s into %s/",

		len(posts), len(pages), len(tagMap), len(catMap), extras, out)

package generator

import (

	"io"

	"log"

	"os"

	"strings"

	"github.com/ruedigerp/newblog/internal/config"

)

// RenderRobotsTxt writes a robots.txt for the site. Without explicit rules the

// whole site is allowed; the sitemap is referenced automatically whenever

// sitemap generation is enabled.

func RenderRobotsTxt(w io.Writer, cfg *config.Config) error {

	bw := &errWriter{w: w}

	bw.printf("User-agent: %s\n", cfg.Robots.UserAgentOrDefault())

	for _, rule := range cfg.Robots.Allow {

		bw.printf("Allow: %s\n", rule)

	}

	for _, rule := range cfg.Robots.Disallow {

		bw.printf("Disallow: %s\n", rule)

	}

	// A crawler needs at least one rule; allow everything when none are configured.

	if len(cfg.Robots.Allow) == 0 && len(cfg.Robots.Disallow) == 0 {

		bw.printf("Allow: /\n")

	}

	if cfg.Sitemap.EnabledOrDefault() && cfg.BaseURL != "" {

		bw.printf("\nSitemap: %s/sitemap.xml\n", strings.TrimRight(cfg.BaseURL, "/"))

	}

	return bw.err

}

// GenerateRobotsTxt writes robots.txt into outDir, honoring cfg.Robots.Enabled

// (defaults to true).

func GenerateRobotsTxt(cfg *config.Config, outDir string) {

	if !cfg.Robots.EnabledOrDefault() {

		return

	}

	if err := writeFile(outDir+"/robots.txt", func(f *os.File) error {

		return RenderRobotsTxt(f, cfg)

	}); err != nil {

		log.Printf("Error writing robots.txt: %v", err)

	}

}

	// sitemap.xml — URL index for search engines

	if cfg.Sitemap.EnabledOrDefault() {

		mux.HandleFunc("/sitemap.xml", func(w http.ResponseWriter, r *http.Request) {

			w.Header().Set("Content-Type", "application/xml; charset=utf-8")

			if err := templates.RenderSitemap(w, site, posts, pages, cfg.PostsPerPage); err != nil {

				log.Printf("Error rendering sitemap: %v", err)

				http.Error(w, "Internal Server Error", 500)

			}

		})

	}

	// robots.txt — crawler rules + sitemap reference

	if cfg.Robots.EnabledOrDefault() {

		mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {

			w.Header().Set("Content-Type", "text/plain; charset=utf-8")

			if err := generator.RenderRobotsTxt(w, cfg); err != nil {

				log.Printf("Error rendering robots.txt: %v", err)

				http.Error(w, "Internal Server Error", 500)

			}

		})

	}

package templates

import (

	"encoding/xml"

	"io"

	"net/url"

	"sort"

	"strconv"

	"strings"

	"time"

	"github.com/ruedigerp/newblog/internal/content"

)

const sitemapNS = "http://www.sitemaps.org/schemas/sitemap/0.9"

type sitemapURL struct {

	XMLName    xml.Name `xml:"url"`

	Loc        string   `xml:"loc"`

	LastMod    string   `xml:"lastmod,omitempty"`

	ChangeFreq string   `xml:"changefreq,omitempty"`

	Priority   string   `xml:"priority,omitempty"`

}

type sitemapURLSet struct {

	XMLName xml.Name     `xml:"urlset"`

	Xmlns   string       `xml:"xmlns,attr"`

	URLs    []sitemapURL `xml:"url"`

}

// RenderSitemap writes a sitemaps.org 0.9 XML sitemap covering the homepage,

// its pagination pages, all posts, all static pages and the tag/category

// listings — i.e. every URL the generator produces as HTML.

func RenderSitemap(w io.Writer, site SiteData, posts []*content.Post, pages []*content.Page, postsPerPage int) error {

	base := strings.TrimRight(site.BaseURL, "/")

	// Newest post date doubles as lastmod for the index pages.

	siteMod := ""

	if len(posts) > 0 {

		siteMod = sitemapDate(postModTime(posts[0]))

	}

	urls := []sitemapURL{{

		Loc:        base + "/",

		LastMod:    siteMod,

		ChangeFreq: "daily",

		Priority:   "1.0",

	}}

	// Pagination: /page/2/, /page/3/, ... (page 1 is the homepage)

	if postsPerPage > 0 {

		totalPages := (len(posts) + postsPerPage - 1) / postsPerPage

		for page := 2; page <= totalPages; page++ {

			urls = append(urls, sitemapURL{

				Loc:        base + "/page/" + strconv.Itoa(page) + "/",

				LastMod:    siteMod,

				ChangeFreq: "weekly",

				Priority:   "0.4",

			})

		}

	}

	// Posts

	for _, p := range posts {

		urls = append(urls, sitemapURL{

			Loc:        base + "/posts/" + escapePath(p.Slug) + "/",

			LastMod:    sitemapDate(postModTime(p)),

			ChangeFreq: "monthly",

			Priority:   "0.8",

		})

	}

	// Static pages

	for _, pg := range pages {

		urls = append(urls, sitemapURL{

			Loc:        base + "/sites/" + escapePath(pg.Slug) + "/",

			ChangeFreq: "monthly",

			Priority:   "0.6",

		})

	}

	// Tag and category listings

	tagMap := content.CollectTags(posts)

	catMap := content.CollectCategories(posts)

	if len(tagMap) > 0 {

		urls = append(urls, sitemapURL{

			Loc:        base + "/tags/",

			LastMod:    siteMod,

			ChangeFreq: "weekly",

			Priority:   "0.5",

		})

		urls = append(urls, taxonomyURLs(base, "tags", tagMap)...)

	}

	if len(catMap) > 0 {

		urls = append(urls, sitemapURL{

			Loc:        base + "/categories/",

			LastMod:    siteMod,

			ChangeFreq: "weekly",

			Priority:   "0.5",

		})

		urls = append(urls, taxonomyURLs(base, "categories", catMap)...)

	}

	set := sitemapURLSet{Xmlns: sitemapNS, URLs: urls}

	if _, err := w.Write([]byte(xml.Header)); err != nil {

		return err

	}

	enc := xml.NewEncoder(w)

	enc.Indent("", "  ")

	if err := enc.Encode(set); err != nil {

		return err

	}

	_, err := w.Write([]byte("\n"))

	return err

}

// taxonomyURLs builds the per-tag / per-category URLs, sorted for stable output.

func taxonomyURLs(base, prefix string, m map[string][]*content.Post) []sitemapURL {

	names := make([]string, 0, len(m))

	for name := range m {

		names = append(names, name)

	}

	sort.Strings(names)

	out := make([]sitemapURL, 0, len(names))

	for _, name := range names {

		lastMod := ""

		for _, p := range m[name] {

			if d := sitemapDate(postModTime(p)); d > lastMod {

				lastMod = d

			}

		}

		out = append(out, sitemapURL{

			// Links in the themes use the lowercased name, same as the generator's dirs.

			Loc:        base + "/" + prefix + "/" + escapePath(strings.ToLower(name)) + "/",

			LastMod:    lastMod,

			ChangeFreq: "weekly",

			Priority:   "0.4",

		})

	}

	return out

}

// postModTime prefers the update date over the publication date.

func postModTime(p *content.Post) time.Time {

	if !p.Update.Time.IsZero() {

		return p.Update.Time

	}

	return p.Date.Time

}

// sitemapDate formats a W3C date (YYYY-MM-DD); zero times yield "".

func sitemapDate(t time.Time) string {

	if t.IsZero() {

		return ""

	}

	return t.Format("2006-01-02")

}

// escapePath percent-encodes each segment of a slug so tags with spaces or

// umlauts stay valid URLs.

func escapePath(slug string) string {

	parts := strings.Split(slug, "/")

	for i, part := range parts {

		parts[i] = url.PathEscape(part)

	}

	return strings.Join(parts, "/")

}