Add bz2 package file decompression, borrow file parsing from aptly, fix mulitple pack version checks

src/deb/format.go

@@ -0,0 +1,307 @@
+package deb
+
+import (
+	"bufio"
+	"errors"
+	"io"
+	"sort"
+	"strings"
+	"unicode"
+)
+
+// Stanza or paragraph of Debian control file
+type Stanza map[string]string
+
+// MaxFieldSize is maximum stanza field size in bytes
+const MaxFieldSize = 2 * 1024 * 1024
+
+// Canonical order of fields in stanza
+// Taken from: http://bazaar.launchpad.net/~ubuntu-branches/ubuntu/vivid/apt/vivid/view/head:/apt-pkg/tagfile.cc#L504
+var (
+	canonicalOrderRelease = []string{
+		"Origin",
+		"Label",
+		"Archive",
+		"Suite",
+		"Version",
+		"Codename",
+		"Date",
+		"NotAutomatic",
+		"ButAutomaticUpgrades",
+		"Architectures",
+		"Architecture",
+		"Components",
+		"Component",
+		"Description",
+		"MD5Sum",
+		"SHA1",
+		"SHA256",
+		"SHA512",
+	}
+
+	canonicalOrderBinary = []string{
+		"Package",
+		"Essential",
+		"Status",
+		"Priority",
+		"Section",
+		"Installed-Size",
+		"Maintainer",
+		"Original-Maintainer",
+		"Architecture",
+		"Source",
+		"Version",
+		"Replaces",
+		"Provides",
+		"Depends",
+		"Pre-Depends",
+		"Recommends",
+		"Suggests",
+		"Conflicts",
+		"Breaks",
+		"Conffiles",
+		"Filename",
+		"Size",
+		"MD5Sum",
+		"MD5sum",
+		"SHA1",
+		"SHA256",
+		"SHA512",
+		"Description",
+	}
+
+	canonicalOrderSource = []string{
+		"Package",
+		"Source",
+		"Binary",
+		"Version",
+		"Priority",
+		"Section",
+		"Maintainer",
+		"Original-Maintainer",
+		"Build-Depends",
+		"Build-Depends-Indep",
+		"Build-Conflicts",
+		"Build-Conflicts-Indep",
+		"Architecture",
+		"Standards-Version",
+		"Format",
+		"Directory",
+		"Files",
+	}
+	canonicalOrderInstaller = []string{
+		"",
+	}
+)
+
+// Copy returns copy of Stanza
+func (s Stanza) Copy() (result Stanza) {
+	result = make(Stanza, len(s))
+	for k, v := range s {
+		result[k] = v
+	}
+	return
+}
+
+func isMultilineField(field string, isRelease bool) bool {
+	switch field {
+	// file without a section
+	case "":
+		return true
+	case "Description":
+		return true
+	case "Files":
+		return true
+	case "Changes":
+		return true
+	case "Checksums-Sha1":
+		return true
+	case "Checksums-Sha256":
+		return true
+	case "Checksums-Sha512":
+		return true
+	case "Package-List":
+		return true
+	case "MD5Sum":
+		return isRelease
+	case "SHA1":
+		return isRelease
+	case "SHA256":
+		return isRelease
+	case "SHA512":
+		return isRelease
+	}
+	return false
+}
+
+// Write single field from Stanza to writer.
+//
+// nolint: interfacer
+func writeField(w *bufio.Writer, field, value string, isRelease bool) (err error) {
+	if !isMultilineField(field, isRelease) {
+		_, err = w.WriteString(field + ": " + value + "\n")
+	} else {
+		if field != "" && !strings.HasSuffix(value, "\n") {
+			value = value + "\n"
+		}
+
+		if field != "Description" && field != "" {
+			value = "\n" + value
+		}
+
+		if field != "" {
+			_, err = w.WriteString(field + ":" + value)
+		} else {
+			_, err = w.WriteString(value)
+		}
+	}
+
+	return
+}
+
+// WriteTo saves stanza back to stream, modifying itself on the fly
+func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease, isInstaller bool) error {
+	canonicalOrder := canonicalOrderBinary
+	if isSource {
+		canonicalOrder = canonicalOrderSource
+	}
+	if isRelease {
+		canonicalOrder = canonicalOrderRelease
+	}
+	if isInstaller {
+		canonicalOrder = canonicalOrderInstaller
+	}
+
+	for _, field := range canonicalOrder {
+		value, ok := s[field]
+		if ok {
+			delete(s, field)
+			err := writeField(w, field, value, isRelease)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	// no extra fields in installer
+	if !isInstaller {
+		// Print extra fields in deterministic order (alphabetical)
+		keys := make([]string, len(s))
+		i := 0
+		for field := range s {
+			keys[i] = field
+			i++
+		}
+		sort.Strings(keys)
+		for _, field := range keys {
+			err := writeField(w, field, s[field], isRelease)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// Parsing errors
+var (
+	ErrMalformedStanza = errors.New("malformed stanza syntax")
+)
+
+func canonicalCase(field string) string {
+	upper := strings.ToUpper(field)
+	switch upper {
+	case "SHA1", "SHA256", "SHA512":
+		return upper
+	case "MD5SUM":
+		return "MD5Sum"
+	case "NOTAUTOMATIC":
+		return "NotAutomatic"
+	case "BUTAUTOMATICUPGRADES":
+		return "ButAutomaticUpgrades"
+	}
+
+	startOfWord := true
+
+	return strings.Map(func(r rune) rune {
+		if startOfWord {
+			startOfWord = false
+			return unicode.ToUpper(r)
+		}
+
+		if r == '-' {
+			startOfWord = true
+		}
+
+		return unicode.ToLower(r)
+	}, field)
+}
+
+// ControlFileReader implements reading of control files stanza by stanza
+type ControlFileReader struct {
+	scanner     *bufio.Scanner
+	isRelease   bool
+	isInstaller bool
+}
+
+// NewControlFileReader creates ControlFileReader, it wraps with buffering
+func NewControlFileReader(r io.Reader, isRelease, isInstaller bool) *ControlFileReader {
+	scnr := bufio.NewScanner(bufio.NewReaderSize(r, 32768))
+	scnr.Buffer(nil, MaxFieldSize)
+
+	return &ControlFileReader{
+		scanner:     scnr,
+		isRelease:   isRelease,
+		isInstaller: isInstaller,
+	}
+}
+
+// ReadStanza reeads one stanza from control file
+func (c *ControlFileReader) ReadStanza() (Stanza, error) {
+	stanza := make(Stanza, 32)
+	lastField := ""
+	lastFieldMultiline := c.isInstaller
+
+	for c.scanner.Scan() {
+		line := c.scanner.Text()
+
+		// Current stanza ends with empty line
+		if line == "" {
+			if len(stanza) > 0 {
+				return stanza, nil
+			}
+			continue
+		}
+
+		if line[0] == ' ' || line[0] == '\t' || c.isInstaller {
+			if lastFieldMultiline {
+				stanza[lastField] += line + "\n"
+			} else {
+				stanza[lastField] += " " + strings.TrimSpace(line)
+			}
+		} else {
+			parts := strings.SplitN(line, ":", 2)
+			if len(parts) != 2 {
+				return nil, ErrMalformedStanza
+			}
+			lastField = canonicalCase(parts[0])
+			lastFieldMultiline = isMultilineField(lastField, c.isRelease)
+			if lastFieldMultiline {
+				stanza[lastField] = parts[1]
+				if parts[1] != "" {
+					stanza[lastField] += "\n"
+				}
+			} else {
+				stanza[lastField] = strings.TrimSpace(parts[1])
+			}
+		}
+	}
+	if err := c.scanner.Err(); err != nil {
+		return nil, err
+	}
+	if len(stanza) > 0 {
+		return stanza, nil
+	}
+	return nil, nil
+}

src/go.mod

@@ -6,4 +6,4 @@ require github.com/ulikunitz/xz v0.5.11
 
 require github.com/klauspost/compress v1.16.7
 
-require pault.ag/go/debian v0.15.0 // indirect
+require pault.ag/go/debian v0.15.0

src/main.go

@@ -1,15 +1,17 @@
 package main
 
 import (
-	"bufio"
 	"fmt"
 	"io"
 	"log"
 	"net/http"
 	"os"
+	"ppp/v2/deb"
 	"strings"
 	"sync"
 
+	"compress/bzip2"
+
 	"github.com/klauspost/compress/gzip"
 
 	"github.com/ulikunitz/xz"
@@ -45,6 +47,10 @@ func processFile(url string) map[string]packageInfo {
 	}
 	defer resp.Body.Close()
 	rdr := io.Reader(resp.Body)
+	if strings.HasSuffix(url, ".bz2") {
+		r := bzip2.NewReader(resp.Body)
+		rdr = r
+	}
 	if strings.HasSuffix(url, ".xz") {
 		r, err := xz.NewReader(resp.Body)
 		if err != nil {
@@ -61,47 +67,38 @@ func processFile(url string) map[string]packageInfo {
 	}
 
 	packages := make(map[string]packageInfo)
-	var currentPackage string
+	sreader := deb.NewControlFileReader(rdr, false, false)
-	scanner := bufio.NewScanner(rdr)
+	for {
-	const maxCapacity = 4096 * 4096
+		stanza, err := sreader.ReadStanza()
-	buf := make([]byte, maxCapacity)
+		if err != nil {
-	scanner.Buffer(buf, maxCapacity)
+			panic(err)
-
-	for scanner.Scan() {
-		line := scanner.Text()
-
-		if strings.HasPrefix(line, "Package: ") {
-			pkName := strings.TrimPrefix(line, "Package: ") + " "
-			_, broken := brokenPackages[pkName]
-			if !broken {
-				currentPackage = pkName
-				packages[currentPackage] = packageInfo{
-					Name: pkName,
-				}
-			} else {
-				currentPackage = ""
-			}
-		} else if strings.HasPrefix(line, "Version: ") && currentPackage != "" {
-			ver, err := version.Parse(strings.TrimPrefix(line, "Version: "))
-			if err != nil {
-				panic(err)
-			}
-			packages[currentPackage] = packageInfo{
-				Name:    currentPackage,
-				Version: ver,
-			}
-		} else if strings.HasPrefix(line, "Filename: ") && currentPackage != "" {
-			packages[currentPackage] = packageInfo{
-				Name:     currentPackage,
-				Version:  packages[currentPackage].Version,
-				FilePath: strings.TrimPrefix(line, "Filename: "),
-			}
 		}
-		if line == "" {
+		if stanza == nil {
-			currentPackage = ""
+			break
 		}
 
+		_, broken := brokenPackages[stanza["Package"]]
+		if broken {
+			continue
+		}
+
+		ver, err := version.Parse(stanza["Version"])
+		if err != nil {
+			panic(err)
+		}
+
+		existingPackage, alreadyExists := packages[stanza["Package"]]
+		if alreadyExists && version.Compare(ver, existingPackage.Version) <= 0 {
+			continue
+		}
+
+		packages[stanza["Package"]] = packageInfo{
+			Name:     stanza["Package"],
+			Version:  ver,
+			FilePath: stanza["Filename"],
+		}
 	}
+
 	return packages
 }
 
@@ -112,13 +109,13 @@ func compare(basePackages map[string]packageInfo, targetPackages map[string]pack
 			if version.Compare(info.Version, baseVersion.Version) > 0 {
 				output[pack] = info
 				if !download {
-					os.Stdout.WriteString(pack)
+					os.Stdout.WriteString(pack + " ")
 				}
 			}
 		} else {
 			output[pack] = info
 			if !download {
-				os.Stdout.WriteString(pack)
+				os.Stdout.WriteString(pack + " ")
 			}
 		}
 	}
@@ -197,38 +194,38 @@ type packageInfo struct {
 }
 
 var brokenPackages = map[string]bool{
-	"libkpim5mbox-data ":               true,
+	"libkpim5mbox-data":               true,
-	"libkpim5identitymanagement-data ": true,
+	"libkpim5identitymanagement-data": true,
-	"libkpim5libkdepim-data ":          true,
+	"libkpim5libkdepim-data":          true,
-	"libkpim5imap-data ":               true,
+	"libkpim5imap-data":               true,
-	"libkpim5ldap-data ":               true,
+	"libkpim5ldap-data":               true,
-	"libkpim5mailimporter-data ":       true,
+	"libkpim5mailimporter-data":       true,
-	"libkpim5mailtransport-data ":      true,
+	"libkpim5mailtransport-data":      true,
-	"libkpim5akonadimime-data ":        true,
+	"libkpim5akonadimime-data":        true,
-	"libkpim5kontactinterface-data ":   true,
+	"libkpim5kontactinterface-data":   true,
-	"libkpim5ksieve-data ":             true,
+	"libkpim5ksieve-data":             true,
-	"libkpim5textedit-data ":           true,
+	"libkpim5textedit-data":           true,
-	"libk3b-data ":                     true,
+	"libk3b-data":                     true,
-	"libkpim5eventviews-data ":         true,
+	"libkpim5eventviews-data":         true,
-	"libkpim5incidenceeditor-data ":    true,
+	"libkpim5incidenceeditor-data":    true,
-	"libkpim5calendarsupport-data ":    true,
+	"libkpim5calendarsupport-data":    true,
-	"libkpim5calendarutils-data ":      true,
+	"libkpim5calendarutils-data":      true,
-	"libkpim5grantleetheme-data ":      true,
+	"libkpim5grantleetheme-data":      true,
-	"libkpim5pkpass-data ":             true,
+	"libkpim5pkpass-data":             true,
-	"libkpim5gapi-data ":               true,
+	"libkpim5gapi-data":               true,
-	"libkpim5akonadisearch-data ":      true,
+	"libkpim5akonadisearch-data":      true,
-	"libkpim5gravatar-data ":           true,
+	"libkpim5gravatar-data":           true,
-	"libkpim5akonadicontact-data ":     true,
+	"libkpim5akonadicontact-data":     true,
-	"libkpim5akonadinotes-data ":       true,
+	"libkpim5akonadinotes-data":       true,
-	"libkpim5libkleo-data ":            true,
+	"libkpim5libkleo-data":            true,
-	"plasma-mobile-tweaks ":            true,
+	"plasma-mobile-tweaks":            true,
-	"libkpim5mime-data ":               true,
+	"libkpim5mime-data":               true,
-	"libkf5textaddons-data ":           true,
+	"libkf5textaddons-data":           true,
-	"libkpim5smtp-data ":               true,
+	"libkpim5smtp-data":               true,
-	"libkpim5tnef-data ":               true,
+	"libkpim5tnef-data":               true,
-	"libkpim5akonadicalendar-data ":    true,
+	"libkpim5akonadicalendar-data":    true,
-	"libkpim5akonadi-data ":            true,
+	"libkpim5akonadi-data":            true,
-	"libnvidia-common-390 ":            true,
+	"libnvidia-common-390":            true,
-	"libnvidia-common-530 ":            true,
+	"libnvidia-common-530":            true,
-	"midisport-firmware ":              true,
+	"midisport-firmware":              true,
 }