Add bz2 package file decompression, borrow file parsing from aptly, fix mulitple pack version checks
This commit is contained in:
parent
eb396c812c
commit
1917e53dff
307
src/deb/format.go
Normal file
307
src/deb/format.go
Normal file
@ -0,0 +1,307 @@
|
||||
package deb
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// Stanza or paragraph of Debian control file
|
||||
type Stanza map[string]string
|
||||
|
||||
// MaxFieldSize is maximum stanza field size in bytes
|
||||
const MaxFieldSize = 2 * 1024 * 1024
|
||||
|
||||
// Canonical order of fields in stanza
|
||||
// Taken from: http://bazaar.launchpad.net/~ubuntu-branches/ubuntu/vivid/apt/vivid/view/head:/apt-pkg/tagfile.cc#L504
|
||||
var (
|
||||
canonicalOrderRelease = []string{
|
||||
"Origin",
|
||||
"Label",
|
||||
"Archive",
|
||||
"Suite",
|
||||
"Version",
|
||||
"Codename",
|
||||
"Date",
|
||||
"NotAutomatic",
|
||||
"ButAutomaticUpgrades",
|
||||
"Architectures",
|
||||
"Architecture",
|
||||
"Components",
|
||||
"Component",
|
||||
"Description",
|
||||
"MD5Sum",
|
||||
"SHA1",
|
||||
"SHA256",
|
||||
"SHA512",
|
||||
}
|
||||
|
||||
canonicalOrderBinary = []string{
|
||||
"Package",
|
||||
"Essential",
|
||||
"Status",
|
||||
"Priority",
|
||||
"Section",
|
||||
"Installed-Size",
|
||||
"Maintainer",
|
||||
"Original-Maintainer",
|
||||
"Architecture",
|
||||
"Source",
|
||||
"Version",
|
||||
"Replaces",
|
||||
"Provides",
|
||||
"Depends",
|
||||
"Pre-Depends",
|
||||
"Recommends",
|
||||
"Suggests",
|
||||
"Conflicts",
|
||||
"Breaks",
|
||||
"Conffiles",
|
||||
"Filename",
|
||||
"Size",
|
||||
"MD5Sum",
|
||||
"MD5sum",
|
||||
"SHA1",
|
||||
"SHA256",
|
||||
"SHA512",
|
||||
"Description",
|
||||
}
|
||||
|
||||
canonicalOrderSource = []string{
|
||||
"Package",
|
||||
"Source",
|
||||
"Binary",
|
||||
"Version",
|
||||
"Priority",
|
||||
"Section",
|
||||
"Maintainer",
|
||||
"Original-Maintainer",
|
||||
"Build-Depends",
|
||||
"Build-Depends-Indep",
|
||||
"Build-Conflicts",
|
||||
"Build-Conflicts-Indep",
|
||||
"Architecture",
|
||||
"Standards-Version",
|
||||
"Format",
|
||||
"Directory",
|
||||
"Files",
|
||||
}
|
||||
canonicalOrderInstaller = []string{
|
||||
"",
|
||||
}
|
||||
)
|
||||
|
||||
// Copy returns copy of Stanza
|
||||
func (s Stanza) Copy() (result Stanza) {
|
||||
result = make(Stanza, len(s))
|
||||
for k, v := range s {
|
||||
result[k] = v
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func isMultilineField(field string, isRelease bool) bool {
|
||||
switch field {
|
||||
// file without a section
|
||||
case "":
|
||||
return true
|
||||
case "Description":
|
||||
return true
|
||||
case "Files":
|
||||
return true
|
||||
case "Changes":
|
||||
return true
|
||||
case "Checksums-Sha1":
|
||||
return true
|
||||
case "Checksums-Sha256":
|
||||
return true
|
||||
case "Checksums-Sha512":
|
||||
return true
|
||||
case "Package-List":
|
||||
return true
|
||||
case "MD5Sum":
|
||||
return isRelease
|
||||
case "SHA1":
|
||||
return isRelease
|
||||
case "SHA256":
|
||||
return isRelease
|
||||
case "SHA512":
|
||||
return isRelease
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Write single field from Stanza to writer.
|
||||
//
|
||||
// nolint: interfacer
|
||||
func writeField(w *bufio.Writer, field, value string, isRelease bool) (err error) {
|
||||
if !isMultilineField(field, isRelease) {
|
||||
_, err = w.WriteString(field + ": " + value + "\n")
|
||||
} else {
|
||||
if field != "" && !strings.HasSuffix(value, "\n") {
|
||||
value = value + "\n"
|
||||
}
|
||||
|
||||
if field != "Description" && field != "" {
|
||||
value = "\n" + value
|
||||
}
|
||||
|
||||
if field != "" {
|
||||
_, err = w.WriteString(field + ":" + value)
|
||||
} else {
|
||||
_, err = w.WriteString(value)
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// WriteTo saves stanza back to stream, modifying itself on the fly
|
||||
func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease, isInstaller bool) error {
|
||||
canonicalOrder := canonicalOrderBinary
|
||||
if isSource {
|
||||
canonicalOrder = canonicalOrderSource
|
||||
}
|
||||
if isRelease {
|
||||
canonicalOrder = canonicalOrderRelease
|
||||
}
|
||||
if isInstaller {
|
||||
canonicalOrder = canonicalOrderInstaller
|
||||
}
|
||||
|
||||
for _, field := range canonicalOrder {
|
||||
value, ok := s[field]
|
||||
if ok {
|
||||
delete(s, field)
|
||||
err := writeField(w, field, value, isRelease)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// no extra fields in installer
|
||||
if !isInstaller {
|
||||
// Print extra fields in deterministic order (alphabetical)
|
||||
keys := make([]string, len(s))
|
||||
i := 0
|
||||
for field := range s {
|
||||
keys[i] = field
|
||||
i++
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, field := range keys {
|
||||
err := writeField(w, field, s[field], isRelease)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Parsing errors
|
||||
var (
|
||||
ErrMalformedStanza = errors.New("malformed stanza syntax")
|
||||
)
|
||||
|
||||
func canonicalCase(field string) string {
|
||||
upper := strings.ToUpper(field)
|
||||
switch upper {
|
||||
case "SHA1", "SHA256", "SHA512":
|
||||
return upper
|
||||
case "MD5SUM":
|
||||
return "MD5Sum"
|
||||
case "NOTAUTOMATIC":
|
||||
return "NotAutomatic"
|
||||
case "BUTAUTOMATICUPGRADES":
|
||||
return "ButAutomaticUpgrades"
|
||||
}
|
||||
|
||||
startOfWord := true
|
||||
|
||||
return strings.Map(func(r rune) rune {
|
||||
if startOfWord {
|
||||
startOfWord = false
|
||||
return unicode.ToUpper(r)
|
||||
}
|
||||
|
||||
if r == '-' {
|
||||
startOfWord = true
|
||||
}
|
||||
|
||||
return unicode.ToLower(r)
|
||||
}, field)
|
||||
}
|
||||
|
||||
// ControlFileReader implements reading of control files stanza by stanza
|
||||
type ControlFileReader struct {
|
||||
scanner *bufio.Scanner
|
||||
isRelease bool
|
||||
isInstaller bool
|
||||
}
|
||||
|
||||
// NewControlFileReader creates ControlFileReader, it wraps with buffering
|
||||
func NewControlFileReader(r io.Reader, isRelease, isInstaller bool) *ControlFileReader {
|
||||
scnr := bufio.NewScanner(bufio.NewReaderSize(r, 32768))
|
||||
scnr.Buffer(nil, MaxFieldSize)
|
||||
|
||||
return &ControlFileReader{
|
||||
scanner: scnr,
|
||||
isRelease: isRelease,
|
||||
isInstaller: isInstaller,
|
||||
}
|
||||
}
|
||||
|
||||
// ReadStanza reeads one stanza from control file
|
||||
func (c *ControlFileReader) ReadStanza() (Stanza, error) {
|
||||
stanza := make(Stanza, 32)
|
||||
lastField := ""
|
||||
lastFieldMultiline := c.isInstaller
|
||||
|
||||
for c.scanner.Scan() {
|
||||
line := c.scanner.Text()
|
||||
|
||||
// Current stanza ends with empty line
|
||||
if line == "" {
|
||||
if len(stanza) > 0 {
|
||||
return stanza, nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if line[0] == ' ' || line[0] == '\t' || c.isInstaller {
|
||||
if lastFieldMultiline {
|
||||
stanza[lastField] += line + "\n"
|
||||
} else {
|
||||
stanza[lastField] += " " + strings.TrimSpace(line)
|
||||
}
|
||||
} else {
|
||||
parts := strings.SplitN(line, ":", 2)
|
||||
if len(parts) != 2 {
|
||||
return nil, ErrMalformedStanza
|
||||
}
|
||||
lastField = canonicalCase(parts[0])
|
||||
lastFieldMultiline = isMultilineField(lastField, c.isRelease)
|
||||
if lastFieldMultiline {
|
||||
stanza[lastField] = parts[1]
|
||||
if parts[1] != "" {
|
||||
stanza[lastField] += "\n"
|
||||
}
|
||||
} else {
|
||||
stanza[lastField] = strings.TrimSpace(parts[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := c.scanner.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(stanza) > 0 {
|
||||
return stanza, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
@ -6,4 +6,4 @@ require github.com/ulikunitz/xz v0.5.11
|
||||
|
||||
require github.com/klauspost/compress v1.16.7
|
||||
|
||||
require pault.ag/go/debian v0.15.0 // indirect
|
||||
require pault.ag/go/debian v0.15.0
|
||||
|
||||
141
src/main.go
141
src/main.go
@ -1,15 +1,17 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"ppp/v2/deb"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"compress/bzip2"
|
||||
|
||||
"github.com/klauspost/compress/gzip"
|
||||
|
||||
"github.com/ulikunitz/xz"
|
||||
@ -45,6 +47,10 @@ func processFile(url string) map[string]packageInfo {
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
rdr := io.Reader(resp.Body)
|
||||
if strings.HasSuffix(url, ".bz2") {
|
||||
r := bzip2.NewReader(resp.Body)
|
||||
rdr = r
|
||||
}
|
||||
if strings.HasSuffix(url, ".xz") {
|
||||
r, err := xz.NewReader(resp.Body)
|
||||
if err != nil {
|
||||
@ -61,47 +67,38 @@ func processFile(url string) map[string]packageInfo {
|
||||
}
|
||||
|
||||
packages := make(map[string]packageInfo)
|
||||
var currentPackage string
|
||||
scanner := bufio.NewScanner(rdr)
|
||||
const maxCapacity = 4096 * 4096
|
||||
buf := make([]byte, maxCapacity)
|
||||
scanner.Buffer(buf, maxCapacity)
|
||||
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
|
||||
if strings.HasPrefix(line, "Package: ") {
|
||||
pkName := strings.TrimPrefix(line, "Package: ") + " "
|
||||
_, broken := brokenPackages[pkName]
|
||||
if !broken {
|
||||
currentPackage = pkName
|
||||
packages[currentPackage] = packageInfo{
|
||||
Name: pkName,
|
||||
}
|
||||
} else {
|
||||
currentPackage = ""
|
||||
}
|
||||
} else if strings.HasPrefix(line, "Version: ") && currentPackage != "" {
|
||||
ver, err := version.Parse(strings.TrimPrefix(line, "Version: "))
|
||||
sreader := deb.NewControlFileReader(rdr, false, false)
|
||||
for {
|
||||
stanza, err := sreader.ReadStanza()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
packages[currentPackage] = packageInfo{
|
||||
Name: currentPackage,
|
||||
Version: ver,
|
||||
}
|
||||
} else if strings.HasPrefix(line, "Filename: ") && currentPackage != "" {
|
||||
packages[currentPackage] = packageInfo{
|
||||
Name: currentPackage,
|
||||
Version: packages[currentPackage].Version,
|
||||
FilePath: strings.TrimPrefix(line, "Filename: "),
|
||||
}
|
||||
}
|
||||
if line == "" {
|
||||
currentPackage = ""
|
||||
if stanza == nil {
|
||||
break
|
||||
}
|
||||
|
||||
_, broken := brokenPackages[stanza["Package"]]
|
||||
if broken {
|
||||
continue
|
||||
}
|
||||
|
||||
ver, err := version.Parse(stanza["Version"])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
existingPackage, alreadyExists := packages[stanza["Package"]]
|
||||
if alreadyExists && version.Compare(ver, existingPackage.Version) <= 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
packages[stanza["Package"]] = packageInfo{
|
||||
Name: stanza["Package"],
|
||||
Version: ver,
|
||||
FilePath: stanza["Filename"],
|
||||
}
|
||||
}
|
||||
|
||||
return packages
|
||||
}
|
||||
|
||||
@ -112,13 +109,13 @@ func compare(basePackages map[string]packageInfo, targetPackages map[string]pack
|
||||
if version.Compare(info.Version, baseVersion.Version) > 0 {
|
||||
output[pack] = info
|
||||
if !download {
|
||||
os.Stdout.WriteString(pack)
|
||||
os.Stdout.WriteString(pack + " ")
|
||||
}
|
||||
}
|
||||
} else {
|
||||
output[pack] = info
|
||||
if !download {
|
||||
os.Stdout.WriteString(pack)
|
||||
os.Stdout.WriteString(pack + " ")
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -197,38 +194,38 @@ type packageInfo struct {
|
||||
}
|
||||
|
||||
var brokenPackages = map[string]bool{
|
||||
"libkpim5mbox-data ": true,
|
||||
"libkpim5identitymanagement-data ": true,
|
||||
"libkpim5libkdepim-data ": true,
|
||||
"libkpim5imap-data ": true,
|
||||
"libkpim5ldap-data ": true,
|
||||
"libkpim5mailimporter-data ": true,
|
||||
"libkpim5mailtransport-data ": true,
|
||||
"libkpim5akonadimime-data ": true,
|
||||
"libkpim5kontactinterface-data ": true,
|
||||
"libkpim5ksieve-data ": true,
|
||||
"libkpim5textedit-data ": true,
|
||||
"libk3b-data ": true,
|
||||
"libkpim5eventviews-data ": true,
|
||||
"libkpim5incidenceeditor-data ": true,
|
||||
"libkpim5calendarsupport-data ": true,
|
||||
"libkpim5calendarutils-data ": true,
|
||||
"libkpim5grantleetheme-data ": true,
|
||||
"libkpim5pkpass-data ": true,
|
||||
"libkpim5gapi-data ": true,
|
||||
"libkpim5akonadisearch-data ": true,
|
||||
"libkpim5gravatar-data ": true,
|
||||
"libkpim5akonadicontact-data ": true,
|
||||
"libkpim5akonadinotes-data ": true,
|
||||
"libkpim5libkleo-data ": true,
|
||||
"plasma-mobile-tweaks ": true,
|
||||
"libkpim5mime-data ": true,
|
||||
"libkf5textaddons-data ": true,
|
||||
"libkpim5smtp-data ": true,
|
||||
"libkpim5tnef-data ": true,
|
||||
"libkpim5akonadicalendar-data ": true,
|
||||
"libkpim5akonadi-data ": true,
|
||||
"libnvidia-common-390 ": true,
|
||||
"libnvidia-common-530 ": true,
|
||||
"midisport-firmware ": true,
|
||||
"libkpim5mbox-data": true,
|
||||
"libkpim5identitymanagement-data": true,
|
||||
"libkpim5libkdepim-data": true,
|
||||
"libkpim5imap-data": true,
|
||||
"libkpim5ldap-data": true,
|
||||
"libkpim5mailimporter-data": true,
|
||||
"libkpim5mailtransport-data": true,
|
||||
"libkpim5akonadimime-data": true,
|
||||
"libkpim5kontactinterface-data": true,
|
||||
"libkpim5ksieve-data": true,
|
||||
"libkpim5textedit-data": true,
|
||||
"libk3b-data": true,
|
||||
"libkpim5eventviews-data": true,
|
||||
"libkpim5incidenceeditor-data": true,
|
||||
"libkpim5calendarsupport-data": true,
|
||||
"libkpim5calendarutils-data": true,
|
||||
"libkpim5grantleetheme-data": true,
|
||||
"libkpim5pkpass-data": true,
|
||||
"libkpim5gapi-data": true,
|
||||
"libkpim5akonadisearch-data": true,
|
||||
"libkpim5gravatar-data": true,
|
||||
"libkpim5akonadicontact-data": true,
|
||||
"libkpim5akonadinotes-data": true,
|
||||
"libkpim5libkleo-data": true,
|
||||
"plasma-mobile-tweaks": true,
|
||||
"libkpim5mime-data": true,
|
||||
"libkf5textaddons-data": true,
|
||||
"libkpim5smtp-data": true,
|
||||
"libkpim5tnef-data": true,
|
||||
"libkpim5akonadicalendar-data": true,
|
||||
"libkpim5akonadi-data": true,
|
||||
"libnvidia-common-390": true,
|
||||
"libnvidia-common-530": true,
|
||||
"midisport-firmware": true,
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user