Add bz2 package file decompression, borrow file parsing from aptly, fix mulitple pack version checks

This commit is contained in:
ferrreo 2023-09-08 19:02:30 +01:00
parent eb396c812c
commit 1917e53dff
4 changed files with 379 additions and 75 deletions

BIN
ppp

Binary file not shown.

307
src/deb/format.go Normal file
View File

@ -0,0 +1,307 @@
package deb
import (
"bufio"
"errors"
"io"
"sort"
"strings"
"unicode"
)
// Stanza or paragraph of Debian control file
type Stanza map[string]string
// MaxFieldSize is maximum stanza field size in bytes
const MaxFieldSize = 2 * 1024 * 1024
// Canonical order of fields in stanza
// Taken from: http://bazaar.launchpad.net/~ubuntu-branches/ubuntu/vivid/apt/vivid/view/head:/apt-pkg/tagfile.cc#L504
var (
canonicalOrderRelease = []string{
"Origin",
"Label",
"Archive",
"Suite",
"Version",
"Codename",
"Date",
"NotAutomatic",
"ButAutomaticUpgrades",
"Architectures",
"Architecture",
"Components",
"Component",
"Description",
"MD5Sum",
"SHA1",
"SHA256",
"SHA512",
}
canonicalOrderBinary = []string{
"Package",
"Essential",
"Status",
"Priority",
"Section",
"Installed-Size",
"Maintainer",
"Original-Maintainer",
"Architecture",
"Source",
"Version",
"Replaces",
"Provides",
"Depends",
"Pre-Depends",
"Recommends",
"Suggests",
"Conflicts",
"Breaks",
"Conffiles",
"Filename",
"Size",
"MD5Sum",
"MD5sum",
"SHA1",
"SHA256",
"SHA512",
"Description",
}
canonicalOrderSource = []string{
"Package",
"Source",
"Binary",
"Version",
"Priority",
"Section",
"Maintainer",
"Original-Maintainer",
"Build-Depends",
"Build-Depends-Indep",
"Build-Conflicts",
"Build-Conflicts-Indep",
"Architecture",
"Standards-Version",
"Format",
"Directory",
"Files",
}
canonicalOrderInstaller = []string{
"",
}
)
// Copy returns copy of Stanza
func (s Stanza) Copy() (result Stanza) {
result = make(Stanza, len(s))
for k, v := range s {
result[k] = v
}
return
}
func isMultilineField(field string, isRelease bool) bool {
switch field {
// file without a section
case "":
return true
case "Description":
return true
case "Files":
return true
case "Changes":
return true
case "Checksums-Sha1":
return true
case "Checksums-Sha256":
return true
case "Checksums-Sha512":
return true
case "Package-List":
return true
case "MD5Sum":
return isRelease
case "SHA1":
return isRelease
case "SHA256":
return isRelease
case "SHA512":
return isRelease
}
return false
}
// Write single field from Stanza to writer.
//
// nolint: interfacer
func writeField(w *bufio.Writer, field, value string, isRelease bool) (err error) {
if !isMultilineField(field, isRelease) {
_, err = w.WriteString(field + ": " + value + "\n")
} else {
if field != "" && !strings.HasSuffix(value, "\n") {
value = value + "\n"
}
if field != "Description" && field != "" {
value = "\n" + value
}
if field != "" {
_, err = w.WriteString(field + ":" + value)
} else {
_, err = w.WriteString(value)
}
}
return
}
// WriteTo saves stanza back to stream, modifying itself on the fly
func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease, isInstaller bool) error {
canonicalOrder := canonicalOrderBinary
if isSource {
canonicalOrder = canonicalOrderSource
}
if isRelease {
canonicalOrder = canonicalOrderRelease
}
if isInstaller {
canonicalOrder = canonicalOrderInstaller
}
for _, field := range canonicalOrder {
value, ok := s[field]
if ok {
delete(s, field)
err := writeField(w, field, value, isRelease)
if err != nil {
return err
}
}
}
// no extra fields in installer
if !isInstaller {
// Print extra fields in deterministic order (alphabetical)
keys := make([]string, len(s))
i := 0
for field := range s {
keys[i] = field
i++
}
sort.Strings(keys)
for _, field := range keys {
err := writeField(w, field, s[field], isRelease)
if err != nil {
return err
}
}
}
return nil
}
// Parsing errors
var (
ErrMalformedStanza = errors.New("malformed stanza syntax")
)
func canonicalCase(field string) string {
upper := strings.ToUpper(field)
switch upper {
case "SHA1", "SHA256", "SHA512":
return upper
case "MD5SUM":
return "MD5Sum"
case "NOTAUTOMATIC":
return "NotAutomatic"
case "BUTAUTOMATICUPGRADES":
return "ButAutomaticUpgrades"
}
startOfWord := true
return strings.Map(func(r rune) rune {
if startOfWord {
startOfWord = false
return unicode.ToUpper(r)
}
if r == '-' {
startOfWord = true
}
return unicode.ToLower(r)
}, field)
}
// ControlFileReader implements reading of control files stanza by stanza
type ControlFileReader struct {
scanner *bufio.Scanner
isRelease bool
isInstaller bool
}
// NewControlFileReader creates ControlFileReader, it wraps with buffering
func NewControlFileReader(r io.Reader, isRelease, isInstaller bool) *ControlFileReader {
scnr := bufio.NewScanner(bufio.NewReaderSize(r, 32768))
scnr.Buffer(nil, MaxFieldSize)
return &ControlFileReader{
scanner: scnr,
isRelease: isRelease,
isInstaller: isInstaller,
}
}
// ReadStanza reeads one stanza from control file
func (c *ControlFileReader) ReadStanza() (Stanza, error) {
stanza := make(Stanza, 32)
lastField := ""
lastFieldMultiline := c.isInstaller
for c.scanner.Scan() {
line := c.scanner.Text()
// Current stanza ends with empty line
if line == "" {
if len(stanza) > 0 {
return stanza, nil
}
continue
}
if line[0] == ' ' || line[0] == '\t' || c.isInstaller {
if lastFieldMultiline {
stanza[lastField] += line + "\n"
} else {
stanza[lastField] += " " + strings.TrimSpace(line)
}
} else {
parts := strings.SplitN(line, ":", 2)
if len(parts) != 2 {
return nil, ErrMalformedStanza
}
lastField = canonicalCase(parts[0])
lastFieldMultiline = isMultilineField(lastField, c.isRelease)
if lastFieldMultiline {
stanza[lastField] = parts[1]
if parts[1] != "" {
stanza[lastField] += "\n"
}
} else {
stanza[lastField] = strings.TrimSpace(parts[1])
}
}
}
if err := c.scanner.Err(); err != nil {
return nil, err
}
if len(stanza) > 0 {
return stanza, nil
}
return nil, nil
}

View File

@ -6,4 +6,4 @@ require github.com/ulikunitz/xz v0.5.11
require github.com/klauspost/compress v1.16.7
require pault.ag/go/debian v0.15.0 // indirect
require pault.ag/go/debian v0.15.0

View File

@ -1,15 +1,17 @@
package main
import (
"bufio"
"fmt"
"io"
"log"
"net/http"
"os"
"ppp/v2/deb"
"strings"
"sync"
"compress/bzip2"
"github.com/klauspost/compress/gzip"
"github.com/ulikunitz/xz"
@ -45,6 +47,10 @@ func processFile(url string) map[string]packageInfo {
}
defer resp.Body.Close()
rdr := io.Reader(resp.Body)
if strings.HasSuffix(url, ".bz2") {
r := bzip2.NewReader(resp.Body)
rdr = r
}
if strings.HasSuffix(url, ".xz") {
r, err := xz.NewReader(resp.Body)
if err != nil {
@ -61,47 +67,38 @@ func processFile(url string) map[string]packageInfo {
}
packages := make(map[string]packageInfo)
var currentPackage string
scanner := bufio.NewScanner(rdr)
const maxCapacity = 4096 * 4096
buf := make([]byte, maxCapacity)
scanner.Buffer(buf, maxCapacity)
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, "Package: ") {
pkName := strings.TrimPrefix(line, "Package: ") + " "
_, broken := brokenPackages[pkName]
if !broken {
currentPackage = pkName
packages[currentPackage] = packageInfo{
Name: pkName,
}
} else {
currentPackage = ""
}
} else if strings.HasPrefix(line, "Version: ") && currentPackage != "" {
ver, err := version.Parse(strings.TrimPrefix(line, "Version: "))
if err != nil {
panic(err)
}
packages[currentPackage] = packageInfo{
Name: currentPackage,
Version: ver,
}
} else if strings.HasPrefix(line, "Filename: ") && currentPackage != "" {
packages[currentPackage] = packageInfo{
Name: currentPackage,
Version: packages[currentPackage].Version,
FilePath: strings.TrimPrefix(line, "Filename: "),
}
sreader := deb.NewControlFileReader(rdr, false, false)
for {
stanza, err := sreader.ReadStanza()
if err != nil {
panic(err)
}
if line == "" {
currentPackage = ""
if stanza == nil {
break
}
_, broken := brokenPackages[stanza["Package"]]
if broken {
continue
}
ver, err := version.Parse(stanza["Version"])
if err != nil {
panic(err)
}
existingPackage, alreadyExists := packages[stanza["Package"]]
if alreadyExists && version.Compare(ver, existingPackage.Version) <= 0 {
continue
}
packages[stanza["Package"]] = packageInfo{
Name: stanza["Package"],
Version: ver,
FilePath: stanza["Filename"],
}
}
return packages
}
@ -112,13 +109,13 @@ func compare(basePackages map[string]packageInfo, targetPackages map[string]pack
if version.Compare(info.Version, baseVersion.Version) > 0 {
output[pack] = info
if !download {
os.Stdout.WriteString(pack)
os.Stdout.WriteString(pack + " ")
}
}
} else {
output[pack] = info
if !download {
os.Stdout.WriteString(pack)
os.Stdout.WriteString(pack + " ")
}
}
}
@ -197,38 +194,38 @@ type packageInfo struct {
}
var brokenPackages = map[string]bool{
"libkpim5mbox-data ": true,
"libkpim5identitymanagement-data ": true,
"libkpim5libkdepim-data ": true,
"libkpim5imap-data ": true,
"libkpim5ldap-data ": true,
"libkpim5mailimporter-data ": true,
"libkpim5mailtransport-data ": true,
"libkpim5akonadimime-data ": true,
"libkpim5kontactinterface-data ": true,
"libkpim5ksieve-data ": true,
"libkpim5textedit-data ": true,
"libk3b-data ": true,
"libkpim5eventviews-data ": true,
"libkpim5incidenceeditor-data ": true,
"libkpim5calendarsupport-data ": true,
"libkpim5calendarutils-data ": true,
"libkpim5grantleetheme-data ": true,
"libkpim5pkpass-data ": true,
"libkpim5gapi-data ": true,
"libkpim5akonadisearch-data ": true,
"libkpim5gravatar-data ": true,
"libkpim5akonadicontact-data ": true,
"libkpim5akonadinotes-data ": true,
"libkpim5libkleo-data ": true,
"plasma-mobile-tweaks ": true,
"libkpim5mime-data ": true,
"libkf5textaddons-data ": true,
"libkpim5smtp-data ": true,
"libkpim5tnef-data ": true,
"libkpim5akonadicalendar-data ": true,
"libkpim5akonadi-data ": true,
"libnvidia-common-390 ": true,
"libnvidia-common-530 ": true,
"midisport-firmware ": true,
"libkpim5mbox-data": true,
"libkpim5identitymanagement-data": true,
"libkpim5libkdepim-data": true,
"libkpim5imap-data": true,
"libkpim5ldap-data": true,
"libkpim5mailimporter-data": true,
"libkpim5mailtransport-data": true,
"libkpim5akonadimime-data": true,
"libkpim5kontactinterface-data": true,
"libkpim5ksieve-data": true,
"libkpim5textedit-data": true,
"libk3b-data": true,
"libkpim5eventviews-data": true,
"libkpim5incidenceeditor-data": true,
"libkpim5calendarsupport-data": true,
"libkpim5calendarutils-data": true,
"libkpim5grantleetheme-data": true,
"libkpim5pkpass-data": true,
"libkpim5gapi-data": true,
"libkpim5akonadisearch-data": true,
"libkpim5gravatar-data": true,
"libkpim5akonadicontact-data": true,
"libkpim5akonadinotes-data": true,
"libkpim5libkleo-data": true,
"plasma-mobile-tweaks": true,
"libkpim5mime-data": true,
"libkf5textaddons-data": true,
"libkpim5smtp-data": true,
"libkpim5tnef-data": true,
"libkpim5akonadicalendar-data": true,
"libkpim5akonadi-data": true,
"libnvidia-common-390": true,
"libnvidia-common-530": true,
"midisport-firmware": true,
}