diff --git a/ppp b/ppp index 9f58ecd..0ccc2ce 100755 Binary files a/ppp and b/ppp differ diff --git a/src/deb/format.go b/src/deb/format.go new file mode 100644 index 0000000..95febe3 --- /dev/null +++ b/src/deb/format.go @@ -0,0 +1,307 @@ +package deb + +import ( + "bufio" + "errors" + "io" + "sort" + "strings" + "unicode" +) + +// Stanza or paragraph of Debian control file +type Stanza map[string]string + +// MaxFieldSize is maximum stanza field size in bytes +const MaxFieldSize = 2 * 1024 * 1024 + +// Canonical order of fields in stanza +// Taken from: http://bazaar.launchpad.net/~ubuntu-branches/ubuntu/vivid/apt/vivid/view/head:/apt-pkg/tagfile.cc#L504 +var ( + canonicalOrderRelease = []string{ + "Origin", + "Label", + "Archive", + "Suite", + "Version", + "Codename", + "Date", + "NotAutomatic", + "ButAutomaticUpgrades", + "Architectures", + "Architecture", + "Components", + "Component", + "Description", + "MD5Sum", + "SHA1", + "SHA256", + "SHA512", + } + + canonicalOrderBinary = []string{ + "Package", + "Essential", + "Status", + "Priority", + "Section", + "Installed-Size", + "Maintainer", + "Original-Maintainer", + "Architecture", + "Source", + "Version", + "Replaces", + "Provides", + "Depends", + "Pre-Depends", + "Recommends", + "Suggests", + "Conflicts", + "Breaks", + "Conffiles", + "Filename", + "Size", + "MD5Sum", + "MD5sum", + "SHA1", + "SHA256", + "SHA512", + "Description", + } + + canonicalOrderSource = []string{ + "Package", + "Source", + "Binary", + "Version", + "Priority", + "Section", + "Maintainer", + "Original-Maintainer", + "Build-Depends", + "Build-Depends-Indep", + "Build-Conflicts", + "Build-Conflicts-Indep", + "Architecture", + "Standards-Version", + "Format", + "Directory", + "Files", + } + canonicalOrderInstaller = []string{ + "", + } +) + +// Copy returns copy of Stanza +func (s Stanza) Copy() (result Stanza) { + result = make(Stanza, len(s)) + for k, v := range s { + result[k] = v + } + return +} + +func isMultilineField(field string, isRelease bool) bool { + switch field { + // file without a section + case "": + return true + case "Description": + return true + case "Files": + return true + case "Changes": + return true + case "Checksums-Sha1": + return true + case "Checksums-Sha256": + return true + case "Checksums-Sha512": + return true + case "Package-List": + return true + case "MD5Sum": + return isRelease + case "SHA1": + return isRelease + case "SHA256": + return isRelease + case "SHA512": + return isRelease + } + return false +} + +// Write single field from Stanza to writer. +// +// nolint: interfacer +func writeField(w *bufio.Writer, field, value string, isRelease bool) (err error) { + if !isMultilineField(field, isRelease) { + _, err = w.WriteString(field + ": " + value + "\n") + } else { + if field != "" && !strings.HasSuffix(value, "\n") { + value = value + "\n" + } + + if field != "Description" && field != "" { + value = "\n" + value + } + + if field != "" { + _, err = w.WriteString(field + ":" + value) + } else { + _, err = w.WriteString(value) + } + } + + return +} + +// WriteTo saves stanza back to stream, modifying itself on the fly +func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease, isInstaller bool) error { + canonicalOrder := canonicalOrderBinary + if isSource { + canonicalOrder = canonicalOrderSource + } + if isRelease { + canonicalOrder = canonicalOrderRelease + } + if isInstaller { + canonicalOrder = canonicalOrderInstaller + } + + for _, field := range canonicalOrder { + value, ok := s[field] + if ok { + delete(s, field) + err := writeField(w, field, value, isRelease) + if err != nil { + return err + } + } + } + + // no extra fields in installer + if !isInstaller { + // Print extra fields in deterministic order (alphabetical) + keys := make([]string, len(s)) + i := 0 + for field := range s { + keys[i] = field + i++ + } + sort.Strings(keys) + for _, field := range keys { + err := writeField(w, field, s[field], isRelease) + if err != nil { + return err + } + } + } + + return nil +} + +// Parsing errors +var ( + ErrMalformedStanza = errors.New("malformed stanza syntax") +) + +func canonicalCase(field string) string { + upper := strings.ToUpper(field) + switch upper { + case "SHA1", "SHA256", "SHA512": + return upper + case "MD5SUM": + return "MD5Sum" + case "NOTAUTOMATIC": + return "NotAutomatic" + case "BUTAUTOMATICUPGRADES": + return "ButAutomaticUpgrades" + } + + startOfWord := true + + return strings.Map(func(r rune) rune { + if startOfWord { + startOfWord = false + return unicode.ToUpper(r) + } + + if r == '-' { + startOfWord = true + } + + return unicode.ToLower(r) + }, field) +} + +// ControlFileReader implements reading of control files stanza by stanza +type ControlFileReader struct { + scanner *bufio.Scanner + isRelease bool + isInstaller bool +} + +// NewControlFileReader creates ControlFileReader, it wraps with buffering +func NewControlFileReader(r io.Reader, isRelease, isInstaller bool) *ControlFileReader { + scnr := bufio.NewScanner(bufio.NewReaderSize(r, 32768)) + scnr.Buffer(nil, MaxFieldSize) + + return &ControlFileReader{ + scanner: scnr, + isRelease: isRelease, + isInstaller: isInstaller, + } +} + +// ReadStanza reeads one stanza from control file +func (c *ControlFileReader) ReadStanza() (Stanza, error) { + stanza := make(Stanza, 32) + lastField := "" + lastFieldMultiline := c.isInstaller + + for c.scanner.Scan() { + line := c.scanner.Text() + + // Current stanza ends with empty line + if line == "" { + if len(stanza) > 0 { + return stanza, nil + } + continue + } + + if line[0] == ' ' || line[0] == '\t' || c.isInstaller { + if lastFieldMultiline { + stanza[lastField] += line + "\n" + } else { + stanza[lastField] += " " + strings.TrimSpace(line) + } + } else { + parts := strings.SplitN(line, ":", 2) + if len(parts) != 2 { + return nil, ErrMalformedStanza + } + lastField = canonicalCase(parts[0]) + lastFieldMultiline = isMultilineField(lastField, c.isRelease) + if lastFieldMultiline { + stanza[lastField] = parts[1] + if parts[1] != "" { + stanza[lastField] += "\n" + } + } else { + stanza[lastField] = strings.TrimSpace(parts[1]) + } + } + } + if err := c.scanner.Err(); err != nil { + return nil, err + } + if len(stanza) > 0 { + return stanza, nil + } + return nil, nil +} diff --git a/src/go.mod b/src/go.mod index c98bfa7..5ab9574 100644 --- a/src/go.mod +++ b/src/go.mod @@ -6,4 +6,4 @@ require github.com/ulikunitz/xz v0.5.11 require github.com/klauspost/compress v1.16.7 -require pault.ag/go/debian v0.15.0 // indirect +require pault.ag/go/debian v0.15.0 diff --git a/src/main.go b/src/main.go index 2ea5c8e..f705497 100644 --- a/src/main.go +++ b/src/main.go @@ -1,15 +1,17 @@ package main import ( - "bufio" "fmt" "io" "log" "net/http" "os" + "ppp/v2/deb" "strings" "sync" + "compress/bzip2" + "github.com/klauspost/compress/gzip" "github.com/ulikunitz/xz" @@ -45,6 +47,10 @@ func processFile(url string) map[string]packageInfo { } defer resp.Body.Close() rdr := io.Reader(resp.Body) + if strings.HasSuffix(url, ".bz2") { + r := bzip2.NewReader(resp.Body) + rdr = r + } if strings.HasSuffix(url, ".xz") { r, err := xz.NewReader(resp.Body) if err != nil { @@ -61,47 +67,38 @@ func processFile(url string) map[string]packageInfo { } packages := make(map[string]packageInfo) - var currentPackage string - scanner := bufio.NewScanner(rdr) - const maxCapacity = 4096 * 4096 - buf := make([]byte, maxCapacity) - scanner.Buffer(buf, maxCapacity) - - for scanner.Scan() { - line := scanner.Text() - - if strings.HasPrefix(line, "Package: ") { - pkName := strings.TrimPrefix(line, "Package: ") + " " - _, broken := brokenPackages[pkName] - if !broken { - currentPackage = pkName - packages[currentPackage] = packageInfo{ - Name: pkName, - } - } else { - currentPackage = "" - } - } else if strings.HasPrefix(line, "Version: ") && currentPackage != "" { - ver, err := version.Parse(strings.TrimPrefix(line, "Version: ")) - if err != nil { - panic(err) - } - packages[currentPackage] = packageInfo{ - Name: currentPackage, - Version: ver, - } - } else if strings.HasPrefix(line, "Filename: ") && currentPackage != "" { - packages[currentPackage] = packageInfo{ - Name: currentPackage, - Version: packages[currentPackage].Version, - FilePath: strings.TrimPrefix(line, "Filename: "), - } + sreader := deb.NewControlFileReader(rdr, false, false) + for { + stanza, err := sreader.ReadStanza() + if err != nil { + panic(err) } - if line == "" { - currentPackage = "" + if stanza == nil { + break } + _, broken := brokenPackages[stanza["Package"]] + if broken { + continue + } + + ver, err := version.Parse(stanza["Version"]) + if err != nil { + panic(err) + } + + existingPackage, alreadyExists := packages[stanza["Package"]] + if alreadyExists && version.Compare(ver, existingPackage.Version) <= 0 { + continue + } + + packages[stanza["Package"]] = packageInfo{ + Name: stanza["Package"], + Version: ver, + FilePath: stanza["Filename"], + } } + return packages } @@ -112,13 +109,13 @@ func compare(basePackages map[string]packageInfo, targetPackages map[string]pack if version.Compare(info.Version, baseVersion.Version) > 0 { output[pack] = info if !download { - os.Stdout.WriteString(pack) + os.Stdout.WriteString(pack + " ") } } } else { output[pack] = info if !download { - os.Stdout.WriteString(pack) + os.Stdout.WriteString(pack + " ") } } } @@ -197,38 +194,38 @@ type packageInfo struct { } var brokenPackages = map[string]bool{ - "libkpim5mbox-data ": true, - "libkpim5identitymanagement-data ": true, - "libkpim5libkdepim-data ": true, - "libkpim5imap-data ": true, - "libkpim5ldap-data ": true, - "libkpim5mailimporter-data ": true, - "libkpim5mailtransport-data ": true, - "libkpim5akonadimime-data ": true, - "libkpim5kontactinterface-data ": true, - "libkpim5ksieve-data ": true, - "libkpim5textedit-data ": true, - "libk3b-data ": true, - "libkpim5eventviews-data ": true, - "libkpim5incidenceeditor-data ": true, - "libkpim5calendarsupport-data ": true, - "libkpim5calendarutils-data ": true, - "libkpim5grantleetheme-data ": true, - "libkpim5pkpass-data ": true, - "libkpim5gapi-data ": true, - "libkpim5akonadisearch-data ": true, - "libkpim5gravatar-data ": true, - "libkpim5akonadicontact-data ": true, - "libkpim5akonadinotes-data ": true, - "libkpim5libkleo-data ": true, - "plasma-mobile-tweaks ": true, - "libkpim5mime-data ": true, - "libkf5textaddons-data ": true, - "libkpim5smtp-data ": true, - "libkpim5tnef-data ": true, - "libkpim5akonadicalendar-data ": true, - "libkpim5akonadi-data ": true, - "libnvidia-common-390 ": true, - "libnvidia-common-530 ": true, - "midisport-firmware ": true, + "libkpim5mbox-data": true, + "libkpim5identitymanagement-data": true, + "libkpim5libkdepim-data": true, + "libkpim5imap-data": true, + "libkpim5ldap-data": true, + "libkpim5mailimporter-data": true, + "libkpim5mailtransport-data": true, + "libkpim5akonadimime-data": true, + "libkpim5kontactinterface-data": true, + "libkpim5ksieve-data": true, + "libkpim5textedit-data": true, + "libk3b-data": true, + "libkpim5eventviews-data": true, + "libkpim5incidenceeditor-data": true, + "libkpim5calendarsupport-data": true, + "libkpim5calendarutils-data": true, + "libkpim5grantleetheme-data": true, + "libkpim5pkpass-data": true, + "libkpim5gapi-data": true, + "libkpim5akonadisearch-data": true, + "libkpim5gravatar-data": true, + "libkpim5akonadicontact-data": true, + "libkpim5akonadinotes-data": true, + "libkpim5libkleo-data": true, + "plasma-mobile-tweaks": true, + "libkpim5mime-data": true, + "libkf5textaddons-data": true, + "libkpim5smtp-data": true, + "libkpim5tnef-data": true, + "libkpim5akonadicalendar-data": true, + "libkpim5akonadi-data": true, + "libnvidia-common-390": true, + "libnvidia-common-530": true, + "midisport-firmware": true, }