Küpper/Quildrop · LabCodeHub

Küpper / Quildrop öffentlich

R Rüdiger Küpper fix: new version with themes + multi arch builds

b56ed7e0 vor 14 Tagen

internal/generator/llms.go 159 Zeilen · 4.9 KB · Go

package generator

import (
	"fmt"
	"io"
	"log"
	"os"
	"strings"

	"github.com/ruedigerp/newblog/internal/config"
	"github.com/ruedigerp/newblog/internal/content"
)

// RenderLLMsTxt writes an llms.txt index pointing LLMs at the site's resources.
// Follows the proposal at https://llmstxt.org/ — H1 title, blockquote summary,
// then H2 sections with markdown link lists.
func RenderLLMsTxt(w io.Writer, cfg *config.Config, posts []*content.Post, pages []*content.Page) error {
	bw := &errWriter{w: w}

	bw.printf("# %s\n\n", cfg.Title)
	bw.printf("> %s\n\n", cfg.Description)
	if cfg.Author != "" {
		bw.printf("Author: %s\n", cfg.Author)
	}
	bw.printf("Site: %s\n\n", cfg.BaseURL)
	bw.printf("Diese Datei folgt der llms.txt-Konvention (https://llmstxt.org/) und listet "+
		"alle Inhalte des Blogs, damit Sprachmodelle sie effizient finden und einordnen können. "+
		"Die vollständigen Texte liegen unter %s/llms-full.txt.\n\n", strings.TrimRight(cfg.BaseURL, "/"))

	// Blog posts
	bw.printf("## Blogposts\n\n")
	for _, p := range posts {
		url := strings.TrimRight(cfg.BaseURL, "/") + "/posts/" + p.Slug + "/"
		preview := strings.TrimSpace(p.GetPreview())
		date := p.Date.Format("2006-01-02")
		if preview != "" {
			bw.printf("- [%s](%s) — %s (%s)\n", p.Title, url, preview, date)
		} else {
			bw.printf("- [%s](%s) (%s)\n", p.Title, url, date)
		}
	}
	bw.printf("\n")

	// Static pages
	if len(pages) > 0 {
		bw.printf("## Seiten\n\n")
		for _, pg := range pages {
			url := strings.TrimRight(cfg.BaseURL, "/") + "/sites/" + pg.Slug + "/"
			if pg.Description != "" {
				bw.printf("- [%s](%s) — %s\n", pg.Title, url, pg.Description)
			} else {
				bw.printf("- [%s](%s)\n", pg.Title, url)
			}
		}
		bw.printf("\n")
	}

	// Optional resources
	base := strings.TrimRight(cfg.BaseURL, "/")
	bw.printf("## Optional\n\n")
	bw.printf("- [Tags](%s/tags/) — Übersicht aller Tags\n", base)
	bw.printf("- [Kategorien](%s/categories/) — Übersicht aller Kategorien\n", base)
	bw.printf("- [RSS Feed](%s/index.xml) — RSS-Feed mit den neuesten Posts\n", base)
	if cfg.LLMs.FullOrDefault() {
		bw.printf("- [Volltext](%s/llms-full.txt) — Alle Blogposts als Markdown in einer Datei\n", base)
	}

	return bw.err
}

// RenderLLMsFullTxt writes llms-full.txt — every post's full markdown content
// concatenated with metadata headers, suitable for LLM ingestion.
func RenderLLMsFullTxt(w io.Writer, cfg *config.Config, posts []*content.Post) error {
	bw := &errWriter{w: w}

	bw.printf("# %s\n\n", cfg.Title)
	bw.printf("> %s\n\n", cfg.Description)
	if cfg.Author != "" {
		bw.printf("Author: %s\n", cfg.Author)
	}
	bw.printf("Site: %s\n\n", cfg.BaseURL)
	if cfg.LLMs.EnabledOrDefault() {
		bw.printf("Dies ist der konsolidierte Volltext aller Blogposts (neueste zuerst). "+
			"Eine knappe Übersicht steht unter %s/llms.txt.\n\n", strings.TrimRight(cfg.BaseURL, "/"))
	} else {
		bw.printf("Dies ist der konsolidierte Volltext aller Blogposts (neueste zuerst).\n\n")
	}

	base := strings.TrimRight(cfg.BaseURL, "/")
	for i, p := range posts {
		if i > 0 {
			bw.printf("\n---\n\n")
		}
		url := base + "/posts/" + p.Slug + "/"
		bw.printf("## %s\n\n", p.Title)
		bw.printf("- URL: %s\n", url)
		bw.printf("- Datum: %s\n", p.Date.Format("2006-01-02"))
		if !p.Update.Time.IsZero() {
			bw.printf("- Aktualisiert: %s\n", p.Update.Format("2006-01-02"))
		}
		if p.Author != "" {
			bw.printf("- Autor: %s\n", p.Author)
		}
		if len(p.Tags) > 0 {
			bw.printf("- Tags: %s\n", strings.Join(p.Tags, ", "))
		}
		if len(p.Categories) > 0 {
			bw.printf("- Kategorien: %s\n", strings.Join(p.Categories, ", "))
		}
		bw.printf("\n")
		// Body is the preprocessed markdown (Hugo shortcodes already stripped).
		body := strings.TrimSpace(p.Content)
		bw.printf("%s\n", body)
	}

	return bw.err
}

// errWriter swallows repeated writes after the first error so callers don't
// have to check every Fprintf.
type errWriter struct {
	w   io.Writer
	err error
}

func (e *errWriter) printf(format string, args ...interface{}) {
	if e.err != nil {
		return
	}
	_, e.err = fmt.Fprintf(e.w, format, args...)
}

// GenerateLLMsFiles writes llms.txt and llms-full.txt into outDir,
// honoring cfg.LLMs.Enabled / cfg.LLMs.Full (both default to true).
func GenerateLLMsFiles(cfg *config.Config, posts []*content.Post, pages []*content.Page, outDir string) {
	if cfg.LLMs.EnabledOrDefault() {
		if err := writeFile(outDir+"/llms.txt", func(f *os.File) error {
			return RenderLLMsTxt(f, cfg, posts, pages)
		}); err != nil {
			log.Printf("Error writing llms.txt: %v", err)
		}
	}
	if cfg.LLMs.FullOrDefault() {
		if err := writeFile(outDir+"/llms-full.txt", func(f *os.File) error {
			return RenderLLMsFullTxt(f, cfg, posts)
		}); err != nil {
			log.Printf("Error writing llms-full.txt: %v", err)
		}
	}
}

func writeFile(path string, render func(f *os.File) error) error {
	f, err := os.Create(path)
	if err != nil {
		return err
	}
	defer f.Close()
	return render(f)
}

1	package generator
2
3	import (
4	"fmt"
5	"io"
6	"log"
7	"os"
8	"strings"
9
10	"github.com/ruedigerp/newblog/internal/config"
11	"github.com/ruedigerp/newblog/internal/content"
12	)
13
14	// RenderLLMsTxt writes an llms.txt index pointing LLMs at the site's resources.
15	// Follows the proposal at https://llmstxt.org/ — H1 title, blockquote summary,
16	// then H2 sections with markdown link lists.
17	func RenderLLMsTxt(w io.Writer, cfg config.Config, posts []content.Post, pages []*content.Page) error {
18	bw := &errWriter{w: w}
19
20	bw.printf("# %s\n\n", cfg.Title)
21	bw.printf("> %s\n\n", cfg.Description)
22	if cfg.Author != "" {
23	bw.printf("Author: %s\n", cfg.Author)
24	}
25	bw.printf("Site: %s\n\n", cfg.BaseURL)
26	bw.printf("Diese Datei folgt der llms.txt-Konvention (https://llmstxt.org/) und listet "+
27	"alle Inhalte des Blogs, damit Sprachmodelle sie effizient finden und einordnen können. "+
28	"Die vollständigen Texte liegen unter %s/llms-full.txt.\n\n", strings.TrimRight(cfg.BaseURL, "/"))
29
30	// Blog posts
31	bw.printf("## Blogposts\n\n")
32	for _, p := range posts {
33	url := strings.TrimRight(cfg.BaseURL, "/") + "/posts/" + p.Slug + "/"
34	preview := strings.TrimSpace(p.GetPreview())
35	date := p.Date.Format("2006-01-02")
36	if preview != "" {
37	bw.printf("- [%s](%s) — %s (%s)\n", p.Title, url, preview, date)
38	} else {
39	bw.printf("- [%s](%s) (%s)\n", p.Title, url, date)
40	}
41	}
42	bw.printf("\n")
43
44	// Static pages
45	if len(pages) > 0 {
46	bw.printf("## Seiten\n\n")
47	for _, pg := range pages {
48	url := strings.TrimRight(cfg.BaseURL, "/") + "/sites/" + pg.Slug + "/"
49	if pg.Description != "" {
50	bw.printf("- [%s](%s) — %s\n", pg.Title, url, pg.Description)
51	} else {
52	bw.printf("- [%s](%s)\n", pg.Title, url)
53	}
54	}
55	bw.printf("\n")
56	}
57
58	// Optional resources
59	base := strings.TrimRight(cfg.BaseURL, "/")
60	bw.printf("## Optional\n\n")
61	bw.printf("- [Tags](%s/tags/) — Übersicht aller Tags\n", base)
62	bw.printf("- [Kategorien](%s/categories/) — Übersicht aller Kategorien\n", base)
63	bw.printf("- [RSS Feed](%s/index.xml) — RSS-Feed mit den neuesten Posts\n", base)
64	if cfg.LLMs.FullOrDefault() {
65	bw.printf("- [Volltext](%s/llms-full.txt) — Alle Blogposts als Markdown in einer Datei\n", base)
66	}
67
68	return bw.err
69	}
70
71	// RenderLLMsFullTxt writes llms-full.txt — every post's full markdown content
72	// concatenated with metadata headers, suitable for LLM ingestion.
73	func RenderLLMsFullTxt(w io.Writer, cfg config.Config, posts []content.Post) error {
74	bw := &errWriter{w: w}
75
76	bw.printf("# %s\n\n", cfg.Title)
77	bw.printf("> %s\n\n", cfg.Description)
78	if cfg.Author != "" {
79	bw.printf("Author: %s\n", cfg.Author)
80	}
81	bw.printf("Site: %s\n\n", cfg.BaseURL)
82	if cfg.LLMs.EnabledOrDefault() {
83	bw.printf("Dies ist der konsolidierte Volltext aller Blogposts (neueste zuerst). "+
84	"Eine knappe Übersicht steht unter %s/llms.txt.\n\n", strings.TrimRight(cfg.BaseURL, "/"))
85	} else {
86	bw.printf("Dies ist der konsolidierte Volltext aller Blogposts (neueste zuerst).\n\n")
87	}
88
89	base := strings.TrimRight(cfg.BaseURL, "/")
90	for i, p := range posts {
91	if i > 0 {
92	bw.printf("\n---\n\n")
93	}
94	url := base + "/posts/" + p.Slug + "/"
95	bw.printf("## %s\n\n", p.Title)
96	bw.printf("- URL: %s\n", url)
97	bw.printf("- Datum: %s\n", p.Date.Format("2006-01-02"))
98	if !p.Update.Time.IsZero() {
99	bw.printf("- Aktualisiert: %s\n", p.Update.Format("2006-01-02"))
100	}
101	if p.Author != "" {
102	bw.printf("- Autor: %s\n", p.Author)
103	}
104	if len(p.Tags) > 0 {
105	bw.printf("- Tags: %s\n", strings.Join(p.Tags, ", "))
106	}
107	if len(p.Categories) > 0 {
108	bw.printf("- Kategorien: %s\n", strings.Join(p.Categories, ", "))
109	}
110	bw.printf("\n")
111	// Body is the preprocessed markdown (Hugo shortcodes already stripped).
112	body := strings.TrimSpace(p.Content)
113	bw.printf("%s\n", body)
114	}
115
116	return bw.err
117	}
118
119	// errWriter swallows repeated writes after the first error so callers don't
120	// have to check every Fprintf.
121	type errWriter struct {
122	w io.Writer
123	err error
124	}
125
126	func (e *errWriter) printf(format string, args ...interface{}) {
127	if e.err != nil {
128	return
129	}
130	_, e.err = fmt.Fprintf(e.w, format, args...)
131	}
132
133	// GenerateLLMsFiles writes llms.txt and llms-full.txt into outDir,
134	// honoring cfg.LLMs.Enabled / cfg.LLMs.Full (both default to true).
135	func GenerateLLMsFiles(cfg config.Config, posts []content.Post, pages []*content.Page, outDir string) {
136	if cfg.LLMs.EnabledOrDefault() {
137	if err := writeFile(outDir+"/llms.txt", func(f *os.File) error {
138	return RenderLLMsTxt(f, cfg, posts, pages)
139	}); err != nil {
140	log.Printf("Error writing llms.txt: %v", err)
141	}
142	}
143	if cfg.LLMs.FullOrDefault() {
144	if err := writeFile(outDir+"/llms-full.txt", func(f *os.File) error {
145	return RenderLLMsFullTxt(f, cfg, posts)
146	}); err != nil {
147	log.Printf("Error writing llms-full.txt: %v", err)
148	}
149	}
150	}
151
152	func writeFile(path string, render func(f *os.File) error) error {
153	f, err := os.Create(path)
154	if err != nil {
155	return err
156	}
157	defer f.Close()
158	return render(f)
159	}