perf: make docker release 10 times faster (#5803)

* build: improve multistage docker build

* build: add dockerfile for ci

* build: add BuildDockerMultiplatform function in build.sh for ci

* ci: change build method

* build: add missing mod download command to the Dockerfile

* build: revert changes made ffmpeg installed

* build: use musl build for docker release

* ci: apply to dev version

* fix: don't login on pr

* fix: don't build_docker_with_aria2 on pr

---------

Co-authored-by: Andy Hsu <i@nn.ci>

.github/workflows/build_docker.yml

@@ -3,6 +3,8 @@ name: build_docker
 on:
   push:
     branches: [ main ]
+  pull_request:
+    branches: [ main ]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -10,42 +12,54 @@ concurrency:
 
 jobs:
   build_docker:
-    name: Build docker
+    name: Build Docker
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v4
+        with:
+          go-version: 'stable'
+
+      - name: Build go binary
+        run: bash build.sh dev docker-multiplatform
+
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v5
         with:
           images: xhofe/alist
-      - name: Replace release with dev
+
-        run: |
-          sed -i 's/release/dev/g' Dockerfile
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
+        
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
+
       - name: Login to DockerHub
+        if: github.event_name == 'push'
         uses: docker/login-action@v3
         with:
           username: xhofe
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+
       - name: Build and push
         id: docker_build
         uses: docker/build-push-action@v5
         with:
           context: .
-          push: true
+          file: Dockerfile.ci
+          push: ${{ github.event_name == 'push' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x
 
   build_docker_with_aria2:
     needs: build_docker
     name: Build docker with aria2
     runs-on: ubuntu-latest
+    if: github.event_name == 'push'
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4

.github/workflows/release_docker.yml

@@ -13,6 +13,13 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - uses: actions/setup-go@v4
+        with:
+          go-version: 'stable'
+
+      - name: Build go binary
+        run: bash build.sh release docker-multiplatform
+
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v5
@@ -36,6 +43,7 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
+          file: Dockerfile.ci
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

Dockerfile

@@ -1,9 +1,11 @@
 FROM alpine:edge as builder
 LABEL stage=go-builder
 WORKDIR /app/
+RUN apk add --no-cache bash curl gcc git go musl-dev
+COPY go.mod go.sum ./
+RUN go mod download
 COPY ./ ./
-RUN apk add --no-cache bash curl gcc git go musl-dev; \
+RUN bash build.sh release docker
-    bash build.sh release docker
 
 FROM alpine:edge
 LABEL MAINTAINER="i@nn.ci"
@@ -11,8 +13,11 @@ VOLUME /opt/alist/data/
 WORKDIR /opt/alist/
 COPY --from=builder /app/bin/alist ./
 COPY entrypoint.sh /entrypoint.sh
-RUN apk add --no-cache bash ca-certificates su-exec tzdata; \
+RUN apk update && \
-    chmod +x /entrypoint.sh
+    apk upgrade --no-cache && \
+    apk add --no-cache bash ca-certificates su-exec tzdata; \
+    chmod +x /entrypoint.sh && \
+    rm -rf /var/cache/apk/*
 ENV PUID=0 PGID=0 UMASK=022
 EXPOSE 5244 5245
-CMD [ "/entrypoint.sh" ]
+CMD [ "/entrypoint.sh" ]

Dockerfile.ci

@@ -0,0 +1,16 @@
+FROM alpine:edge
+ARG TARGETPLATFORM
+LABEL MAINTAINER="i@nn.ci"
+VOLUME /opt/alist/data/
+WORKDIR /opt/alist/
+COPY /${TARGETPLATFORM}/alist ./
+COPY entrypoint.sh /entrypoint.sh
+RUN apk update && \
+    apk upgrade --no-cache && \
+    apk add --no-cache bash ca-certificates su-exec tzdata; \
+    chmod +x /entrypoint.sh && \
+    rm -rf /var/cache/apk/* && \
+    /entrypoint.sh version
+ENV PUID=0 PGID=0 UMASK=022
+EXPOSE 5244 5245
+CMD [ "/entrypoint.sh" ]

build.sh

@@ -85,12 +85,61 @@ BuildDev() {
   cat md5.txt
 }
 
-BuildDocker() {
+PrepareBuildDocker() {
   echo "replace github.com/mattn/go-sqlite3 => github.com/leso-kn/go-sqlite3 v0.0.0-20230710125852-03158dc838ed" >>go.mod
   go get gorm.io/driver/sqlite@v1.4.4
+  go mod download
+}
+
+BuildDocker() {
+  PrepareBuildDocker
   go build -o ./bin/alist -ldflags="$ldflags" -tags=jsoniter .
 }
 
+BuildDockerMultiplatform() {
+  PrepareBuildDocker
+
+  BASE="https://musl.cc/"
+  FILES=(x86_64-linux-musl-cross aarch64-linux-musl-cross i486-linux-musl-cross s390x-linux-musl-cross armv6-linux-musleabihf-cross armv7l-linux-musleabihf-cross)
+  for i in "${FILES[@]}"; do
+    url="${BASE}${i}.tgz"
+    curl -L -o "${i}.tgz" "${url}"
+    sudo tar xf "${i}.tgz" --strip-components 1 -C /usr/local
+    rm -f "${i}.tgz"
+  done
+
+  docker_lflags="--extldflags '-static -fpic' $ldflags"
+  export CGO_ENABLED=1
+
+  OS_ARCHES=(linux-amd64 linux-arm64 linux-386 linux-s390x)
+  CGO_ARGS=(x86_64-linux-musl-gcc aarch64-linux-musl-gcc i486-linux-musl-gcc s390x-linux-musl-gcc)
+  for i in "${!OS_ARCHES[@]}"; do
+    os_arch=${OS_ARCHES[$i]}
+    cgo_cc=${CGO_ARGS[$i]}
+    os=${os_arch%%-*}
+    arch=${os_arch##*-}
+    export GOOS=$os
+    export GOARCH=$arch
+    export CC=${cgo_cc}
+    echo "building for $os_arch"
+    go build -o ./$os/$arch/alist -ldflags="$docker_lflags" -tags=jsoniter .
+  done
+
+  DOCKER_ARM_ARCHES=(linux-arm/v6 linux-arm/v7)
+  CGO_ARGS=(armv6-linux-musleabihf-gcc armv7l-linux-musleabihf-gcc)
+  GO_ARM=(6 7)
+  export GOOS=linux
+  export GOARCH=arm
+  for i in "${!DOCKER_ARM_ARCHES[@]}"; do
+    docker_arch=${DOCKER_ARM_ARCHES[$i]}
+    cgo_cc=${CGO_ARGS[$i]}
+    export GOARM=${GO_ARM[$i]}
+    export CC=${cgo_cc}
+    echo "building for $docker_arch"
+    go build -o ./${docker_arch%%-*}/${docker_arch##*-}/alist -ldflags="$docker_lflags" -tags=jsoniter .
+  done
+}
+
 BuildRelease() {
   rm -rf .git/
   mkdir -p "build"
@@ -190,6 +239,8 @@ if [ "$1" = "dev" ]; then
   FetchWebDev
   if [ "$2" = "docker" ]; then
     BuildDocker
+  elif [ "$2" = "docker-multiplatform" ]; then
+      BuildDockerMultiplatform
   else
     BuildDev
   fi
@@ -197,6 +248,8 @@ elif [ "$1" = "release" ]; then
   FetchWebRelease
   if [ "$2" = "docker" ]; then
     BuildDocker
+  elif [ "$2" = "docker-multiplatform" ]; then
+    BuildDockerMultiplatform
   elif [ "$2" = "linux_musl_arm" ]; then
     BuildReleaseLinuxMuslArm
     MakeRelease "md5-linux-musl-arm.txt"

drivers/vtencent/util.go

@@ -100,27 +100,35 @@ func (d *Vtencent) LoadUser() (string, error) {
 }
 
 func (d *Vtencent) GetFiles(dirId string) ([]File, error) {
-	api := "https://api.vs.tencent.com/PaaS/Material/SearchResource"
+	var res []File
-	form := fmt.Sprintf(`{
+	//offset := 0
+	for {
+		api := "https://api.vs.tencent.com/PaaS/Material/SearchResource"
+		form := fmt.Sprintf(`{
 		"Text":"",
 		"Text":"",
-		"Offset":0,
+		"Offset":%d,
-		"Limit":20000,
+		"Limit":50,
 		"Sort":{"Field":"%s","Order":"%s"},
 		"CreateTimeRanges":[],
 		"MaterialTypes":[],
 		"ReviewStatuses":[],
 		"Tags":[],
 		"SearchScopes":[{"Owner":{"Type":"PERSON","Id":"%s"},"ClassId":%s,"SearchOneDepth":true}]
-	}`, d.Addition.OrderBy, d.Addition.OrderDirection, d.TfUid, dirId)
+	}`, len(res), d.Addition.OrderBy, d.Addition.OrderDirection, d.TfUid, dirId)
-	var resps RspFiles
+		var resp RspFiles
-	_, err := d.request(api, http.MethodPost, func(req *resty.Request) {
+		_, err := d.request(api, http.MethodPost, func(req *resty.Request) {
-		req.SetBody(form).ForceContentType("application/json")
+			req.SetBody(form).ForceContentType("application/json")
-	}, &resps)
+		}, &resp)
-	if err != nil {
+		if err != nil {
-		return []File{}, err
+			return nil, err
+		}
+		res = append(res, resp.Data.ResourceInfoSet...)
+		if len(resp.Data.ResourceInfoSet) <= 0 || len(res) >= resp.Data.TotalCount {
+			break
+		}
 	}
-	return resps.Data.ResourceInfoSet, nil
+	return res, nil
 }
 
 func (d *Vtencent) CreateUploadMaterial(classId int, fileName string, UploadSummaryKey string) (RspCreatrMaterial, error) {

entrypoint.sh

@@ -4,4 +4,8 @@ chown -R ${PUID}:${PGID} /opt/alist/
 
 umask ${UMASK}
 
-exec su-exec ${PUID}:${PGID} ./alist server --no-prefix
+if [ "$1" = "version" ]; then
+  ./alist version
+else
+  exec su-exec ${PUID}:${PGID} ./alist server --no-prefix
+fi

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/Xhofe/wopan-sdk-go v0.1.2
 	github.com/aliyun/aliyun-oss-go-sdk v2.2.10+incompatible
 	github.com/avast/retry-go v3.0.0+incompatible
-	github.com/aws/aws-sdk-go v1.49.13
+	github.com/aws/aws-sdk-go v1.49.15
 	github.com/blevesearch/bleve/v2 v2.3.10
 	github.com/caarlos0/env/v9 v9.0.0
 	github.com/charmbracelet/bubbles v0.17.1
@@ -51,8 +51,8 @@ require (
 	github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5
 	github.com/xhofe/tache v0.1.1
 	golang.org/x/crypto v0.17.0
-	golang.org/x/exp v0.0.0-20231226003508-02704c960a9b
+	golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc
-	golang.org/x/image v0.14.0
+	golang.org/x/image v0.15.0
 	golang.org/x/net v0.19.0
 	golang.org/x/oauth2 v0.15.0
 	golang.org/x/time v0.5.0

go.sum

@@ -36,6 +36,8 @@ github.com/aws/aws-sdk-go v1.46.7 h1:IjvAWeiJZlbETOemOwvheN5L17CvKvKW0T1xOC6d3Sc
 github.com/aws/aws-sdk-go v1.46.7/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/aws/aws-sdk-go v1.49.13 h1:f4mGztsgnx2dR9r8FQYa9YW/RsKb+N7bgef4UGrOW1Y=
 github.com/aws/aws-sdk-go v1.49.13/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
+github.com/aws/aws-sdk-go v1.49.15 h1:aH9bSV4kL4ziH0AMtuYbukGIVebXddXBL0cKZ1zj15k=
+github.com/aws/aws-sdk-go v1.49.15/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
 github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
@@ -527,11 +529,15 @@ golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
 golang.org/x/exp v0.0.0-20231226003508-02704c960a9b h1:kLiC65FbiHWFAOu+lxwNPujcsl8VYyTYYEZnsOO1WK4=
 golang.org/x/exp v0.0.0-20231226003508-02704c960a9b/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
+golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM=
+golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
 golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/image v0.11.0 h1:ds2RoQvBvYTiJkwpSFDwCcDFNX7DqjL2WsUgTNk0Ooo=
 golang.org/x/image v0.11.0/go.mod h1:bglhjqbqVuEb9e9+eNR45Jfu7D+T4Qan+NhQk8Ck2P8=
 golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4=
 golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE=
+golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8=
+golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=

internal/net/request.go

@@ -449,16 +449,20 @@ type Buf struct {
 	size   int //expected size
 	ctx    context.Context
 	off    int
-	rw     sync.RWMutex
+	rw     sync.Mutex
-	notify chan struct{}
+	//notify chan struct{}
 }
 
 // NewBuf is a buffer that can have 1 read & 1 write at the same time.
 // when read is faster write, immediately feed data to read after written
 func NewBuf(ctx context.Context, maxSize int, id int) *Buf {
 	d := make([]byte, 0, maxSize)
-	return &Buf{ctx: ctx, buffer: bytes.NewBuffer(d), size: maxSize, notify: make(chan struct{})}
+	return &Buf{
-
+		ctx:    ctx,
+		buffer: bytes.NewBuffer(d),
+		size:   maxSize,
+		//notify: make(chan struct{}),
+	}
 }
 func (br *Buf) Reset(size int) {
 	br.buffer.Reset()
@@ -476,9 +480,9 @@ func (br *Buf) Read(p []byte) (n int, err error) {
 	if br.off >= br.size {
 		return 0, io.EOF
 	}
-	br.rw.RLock()
+	br.rw.Lock()
 	n, err = br.buffer.Read(p)
-	br.rw.RUnlock()
+	br.rw.Unlock()
 	if err == nil {
 		br.off += n
 		return n, err
@@ -495,8 +499,8 @@ func (br *Buf) Read(p []byte) (n int, err error) {
 	select {
 	case <-br.ctx.Done():
 		return 0, br.ctx.Err()
-	case <-br.notify:
+	//case <-br.notify:
-		return 0, nil
+	//	return 0, nil
 	case <-time.After(time.Millisecond * 200):
 		return 0, nil
 	}
@@ -510,12 +514,12 @@ func (br *Buf) Write(p []byte) (n int, err error) {
 	defer br.rw.Unlock()
 	n, err = br.buffer.Write(p)
 	select {
-	case br.notify <- struct{}{}:
+	//case br.notify <- struct{}{}:
 	default:
 	}
 	return
 }
 
 func (br *Buf) Close() {
-	close(br.notify)
+	//close(br.notify)
 }