GeezerVison

Introduction

Installation

CLI

go install github.com/dotcommander/defuddle/cmd/defuddle@latest

JavaScript rendering (opt-in)

defuddle parse --render https://example.com/spa-article

Library

go get github.com/dotcommander/defuddle

Quick Start

CLI

defuddle parse https://example.com/article

defuddle parse https://example.com/article --markdown
defuddle parse https://example.com/article --json

Library — fetch and parse a URL

import (
    "context"
    "fmt"
    "github.com/dotcommander/defuddle"
)

result, err := defuddle.ParseFromURL(context.Background(), "https://example.com/article", nil)
if err != nil {
    log.Fatal(err)
}
fmt.Println(result.Title)
fmt.Println(result.Content) // clean HTML

Library — parse HTML you already have

result, err := defuddle.ParseFromString(ctx, htmlString, &defuddle.Options{
    URL: "https://example.com/article", // enables relative URL resolution
})

Lower-level API

d, err := defuddle.NewDefuddle(htmlString, &defuddle.Options{
    URL:      "https://example.com/article",
    Markdown: true,
})
if err != nil {
    log.Fatal(err)
}

result, err := d.Parse(ctx)

fmt.Printf("Title:       %s\n", result.Title)
fmt.Printf("Author:      %s\n", result.Author)
fmt.Printf("Published:   %s\n", result.Published)
fmt.Printf("Language:    %s\n", result.Language)
fmt.Printf("Word Count:  %d\n", result.WordCount)
fmt.Printf("Content:     %s\n", result.Content) // Markdown when Markdown: true

Extracting Content

From a URL

result, err := defuddle.ParseFromURL(ctx, "https://example.com/article", &defuddle.Options{
    Markdown: true,
})

Multiple URLs (concurrent)

urls := []string{
    "https://example.com/article-1",
    "https://example.com/article-2",
}

results := defuddle.ParseFromURLs(ctx, urls, &defuddle.Options{
    MaxConcurrency: 10,
    Markdown:       true,
})

for _, r := range results {
    if r.Err != nil {
        log.Printf("failed %s: %v", r.URL, r.Err)
        continue
    }
    fmt.Printf("%s (%d words)\n", r.Result.Title, r.Result.WordCount)
}

Markdown Output

result, err := defuddle.ParseFromURL(ctx, url, &defuddle.Options{Markdown: true})
fmt.Println(result.Content) // Markdown

result, err := defuddle.ParseFromURL(ctx, url, &defuddle.Options{SeparateMarkdown: true})
fmt.Println(result.Content)          // HTML
fmt.Println(*result.ContentMarkdown) // Markdown

Site-Specific Extractors

Custom Extractors

type RecipeExtractor struct {
    *extractors.ExtractorBase
}

func NewRecipeExtractor(doc *goquery.Document, url string, schema any) extractors.BaseExtractor {
    return &RecipeExtractor{ExtractorBase: extractors.NewExtractorBase(doc, url, schema)}
}

func (e *RecipeExtractor) Name() string { return "RecipeExtractor" }

// CanExtract returns true only when the page has a recipe card — not every page on the host.
func (e *RecipeExtractor) CanExtract() bool {
    return e.GetDocument().Find("article.recipe-card").Length() > 0
}

func (e *RecipeExtractor) Extract() *extractors.ExtractorResult {
    doc := e.GetDocument()

    // ContentHTML is what becomes Result.Content.
    content, _ := doc.Find("article.recipe-card").Html()

    title := strings.TrimSpace(doc.Find("h1.recipe-title").Text())
    author := strings.TrimSpace(doc.Find(".recipe-author").Text())

    return &extractors.ExtractorResult{
        ContentHTML: content,
        Variables: map[string]string{
            "title":  title,
            "author": author,
            "site":   "Recipe Site",
        },
    }
}

extractors.Register(extractors.ExtractorMapping{
    Patterns:  []any{"recipes.example.com"},
    Extractor: NewRecipeExtractor,
})

Configuration

Options

opts := &defuddle.Options{
    // Output
    Markdown:         false, // Return content as Markdown
    SeparateMarkdown: false, // Return both HTML and Markdown

    // Content selection
    ContentSelector:  "",    // CSS selector override for main content
    URL:              "",    // Source URL (used for link resolution and domain detection)

    // Deprecated removal controls: individual values no longer affect extraction.
    // Set all five to PtrBool(false) to bypass extraction.
    RemoveExactSelectors:   nil,
    RemovePartialSelectors: nil,
    RemoveHiddenElements:   nil,
    RemoveContentPatterns:  nil,
    RemoveLowScoring:       nil,
    RemoveImages:           false, // Strip all images from output

    // Element processing
    ProcessCode:      false, // Normalize code blocks with language detection
    ProcessImages:    false, // Optimize images (lazy-load resolution, srcset)
    ProcessHeadings:  false, // Clean heading hierarchy
    ProcessMath:      false, // Normalize MathJax/KaTeX formulas
    ProcessFootnotes: false, // Standardize footnote format
    ProcessRoles:     false, // Convert ARIA roles to semantic HTML

    // HTTP (for ParseFromURL / ParseFromURLs)
    Client:         nil, // Custom *http.Client; use its transport, jar, and timeout
    Headers:        nil, // Request headers cloned onto each fetch
    MaxConcurrency: 5,   // Parallel limit for ParseFromURLs
    Debug:          false,
}

Content Selector

result, err := defuddle.ParseFromURL(ctx, url, &defuddle.Options{
    ContentSelector: "article.post-body",
})

The Extraction Pipeline

HTML Input
 |
 v
1. Preparation        -- Flatten shadow roots, resolve React SSR, extract metadata
2. Explicit Selection -- Format the first matching selector or bypassed body
3. Site Detection     -- Dispatch a matching specialized extractor
4. Rich Preparation   -- Run enabled processors, save recovery body, project rich nodes
5. Trafilatura        -- Extract generic content with native fallback
6. Rich Restoration   -- Restore selected code, math, and referenced footnotes
7. Recovery           -- Use saved body if extraction or marker validation fails
8. Finalization       -- Resolve URLs, sanitize, serialize, count words
9. Markdown           -- Convert to Markdown (if requested)
 |
 v
Result

The Result Object

CLI Usage

Extracting Content

# From a URL
defuddle parse https://example.com/article

# From a local file
defuddle parse article.html

# From stdin (pipe HTML in)
curl -s https://example.com/article | defuddle parse

# As Markdown
defuddle parse https://example.com/article --markdown

# As JSON with all metadata
defuddle parse https://example.com/article --json

# Extract a single field
defuddle parse https://example.com/article --property title

Batch Processing

defuddle batch < urls.txt > articles.jsonl

# From a file, with markdown, 10 parallel fetches
defuddle batch --input urls.txt --markdown --concurrency 10 > articles.jsonl

# Bound total batch duration; --continue-on-error emits per-line error objects
defuddle batch --input urls.txt --timeout 2m --continue-on-error > articles.jsonl

Saving Output

defuddle parse https://example.com/article --markdown --output article.md

Authentication and Proxies

# Custom headers
defuddle parse https://example.com --header "Authorization: Bearer token123"

# Through a proxy
defuddle parse https://example.com --proxy http://localhost:8080

# Custom timeout
defuddle parse https://slow-site.com --timeout 120s

All CLI Options

Documentation

Limitations

Examples

go run ./examples/basic              # Simple extraction
go run ./examples/markdown           # HTML to Markdown
go run ./examples/advanced           # Full option usage
go run ./examples/extractors         # Site-specific extraction
go run ./examples/custom_extractor   # Building a custom extractor

Testing

# Run all tests
go test ./...

# With race detection
go test -race ./...

# Benchmarks
go test -bench=. -benchmem ./...

Credits

License

Generic extraction engine