Skip to content

Commit 13cbca6

Browse files
mtojekmtojek
authored
feat: support template bundles as zip archives (#11839)
1 parent b25deaa commit 13cbca6

8 files changed

Lines changed: 248 additions & 18 deletions

File tree

‎coderd/apidoc/docs.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/apidoc/swagger.json‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/files.go‎

Lines changed: 61 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
package coderd
22

33
import (
4+
"archive/tar"
5+
"archive/zip"
6+
"bytes"
47
"crypto/sha256"
58
"database/sql"
69
"encoding/hex"
@@ -9,6 +12,7 @@ import (
912
"io"
1013
"net/http"
1114

15+
"cdr.dev/slog"
1216
"github.com/go-chi/chi/v5"
1317
"github.com/google/uuid"
1418

@@ -21,6 +25,9 @@ import (
2125

2226
const (
2327
tarMimeType = "application/x-tar"
28+
zipMimeType = "application/zip"
29+
30+
httpFileMaxBytes = 10 * (10 << 20)
2431
)
2532

2633
// @Summary Upload file
@@ -30,7 +37,7 @@ const (
3037
// @Produce json
3138
// @Accept application/x-tar
3239
// @Tags Files
33-
// @Param Content-Type header string true "Content-Type must be `application/x-tar`" default(application/x-tar)
40+
// @Param Content-Type header string true "Content-Type must be `application/x-tar` or `application/zip`" default(application/x-tar)
3441
// @Param file formData file true "File to be uploaded"
3542
// @Success 201 {object} codersdk.UploadResponse
3643
// @Router /files [post]
@@ -39,17 +46,16 @@ func (api *API) postFile(rw http.ResponseWriter, r *http.Request) {
3946
apiKey := httpmw.APIKey(r)
4047

4148
contentType := r.Header.Get("Content-Type")
42-
4349
switch contentType {
44-
case tarMimeType:
50+
case tarMimeType, zipMimeType:
4551
default:
4652
httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
4753
Message: fmt.Sprintf("Unsupported content type header %q.", contentType),
4854
})
4955
return
5056
}
5157

52-
r.Body = http.MaxBytesReader(rw, r.Body, 10*(10<<20))
58+
r.Body = http.MaxBytesReader(rw, r.Body, httpFileMaxBytes)
5359
data, err := io.ReadAll(r.Body)
5460
if err != nil {
5561
httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
@@ -58,6 +64,28 @@ func (api *API) postFile(rw http.ResponseWriter, r *http.Request) {
5864
})
5965
return
6066
}
67+
68+
if contentType == zipMimeType {
69+
zipReader, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
70+
if err != nil {
71+
httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
72+
Message: "Incomplete .zip archive file.",
73+
Detail: err.Error(),
74+
})
75+
return
76+
}
77+
78+
data, err = CreateTarFromZip(zipReader)
79+
if err != nil {
80+
httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
81+
Message: "Internal error processing .zip archive.",
82+
Detail: err.Error(),
83+
})
84+
return
85+
}
86+
contentType = tarMimeType
87+
}
88+
6189
hashBytes := sha256.Sum256(data)
6290
hash := hex.EncodeToString(hashBytes[:])
6391
file, err := api.Database.GetFileByHashAndCreator(ctx, database.GetFileByHashAndCreatorParams{
@@ -108,7 +136,10 @@ func (api *API) postFile(rw http.ResponseWriter, r *http.Request) {
108136
// @Success 200
109137
// @Router /files/{fileID} [get]
110138
func (api *API) fileByID(rw http.ResponseWriter, r *http.Request) {
111-
ctx := r.Context()
139+
var (
140+
ctx = r.Context()
141+
format = r.URL.Query().Get("format")
142+
)
112143

113144
fileID := chi.URLParam(r, "fileID")
114145
if fileID == "" {
@@ -139,7 +170,29 @@ func (api *API) fileByID(rw http.ResponseWriter, r *http.Request) {
139170
return
140171
}
141172

142-
rw.Header().Set("Content-Type", file.Mimetype)
143-
rw.WriteHeader(http.StatusOK)
144-
_, _ = rw.Write(file.Data)
173+
switch format {
174+
case codersdk.FormatZip:
175+
if file.Mimetype != codersdk.ContentTypeTar {
176+
httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
177+
Message: "Only .tar files can be converted to .zip format",
178+
Detail: err.Error(),
179+
})
180+
return
181+
}
182+
183+
rw.Header().Set("Content-Type", codersdk.ContentTypeZip)
184+
rw.WriteHeader(http.StatusOK)
185+
err = WriteZipArchive(rw, tar.NewReader(bytes.NewReader(file.Data)))
186+
if err != nil {
187+
api.Logger.Error(ctx, "invalid .zip archive", slog.F("file_id", fileID), slog.F("mimetype", file.Mimetype), slog.Error(err))
188+
}
189+
case "": // no format? no conversion
190+
rw.Header().Set("Content-Type", file.Mimetype)
191+
_, _ = rw.Write(file.Data)
192+
default:
193+
httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
194+
Message: "Unsupported conversion format.",
195+
Detail: err.Error(),
196+
})
197+
}
145198
}

‎coderd/files_test.go‎

Lines changed: 68 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package coderd_test
22

33
import (
4+
"archive/tar"
45
"bytes"
56
"context"
67
"net/http"
@@ -9,8 +10,10 @@ import (
910
"github.com/google/uuid"
1011
"github.com/stretchr/testify/require"
1112

13+
"github.com/coder/coder/v2/coderd"
1214
"github.com/coder/coder/v2/coderd/coderdtest"
1315
"github.com/coder/coder/v2/codersdk"
16+
"github.com/coder/coder/v2/provisioner/echo"
1417
"github.com/coder/coder/v2/testutil"
1518
)
1619

@@ -72,19 +75,83 @@ func TestDownload(t *testing.T) {
7275
require.Equal(t, http.StatusNotFound, apiErr.StatusCode())
7376
})
7477

75-
t.Run("Insert", func(t *testing.T) {
78+
t.Run("InsertTar_DownloadTar", func(t *testing.T) {
7679
t.Parallel()
7780
client := coderdtest.New(t, nil)
7881
_ = coderdtest.CreateFirstUser(t, client)
7982

83+
// given
8084
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
8185
defer cancel()
8286

87+
// when
8388
resp, err := client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader(make([]byte, 1024)))
8489
require.NoError(t, err)
8590
data, contentType, err := client.Download(ctx, resp.ID)
8691
require.NoError(t, err)
92+
93+
// then
8794
require.Len(t, data, 1024)
8895
require.Equal(t, codersdk.ContentTypeTar, contentType)
8996
})
97+
98+
t.Run("InsertZip_DownloadTar", func(t *testing.T) {
99+
t.Parallel()
100+
client := coderdtest.New(t, nil)
101+
_ = coderdtest.CreateFirstUser(t, client)
102+
103+
// given
104+
tarball, err := echo.Tar(&echo.Responses{
105+
Parse: echo.ParseComplete,
106+
ProvisionApply: echo.ApplyComplete,
107+
})
108+
require.NoError(t, err)
109+
110+
tarReader := tar.NewReader(bytes.NewReader(tarball))
111+
zipContent, err := coderd.CreateZipFromTar(tarReader)
112+
require.NoError(t, err)
113+
114+
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
115+
defer cancel()
116+
117+
// when
118+
resp, err := client.Upload(ctx, codersdk.ContentTypeZip, bytes.NewReader(zipContent))
119+
require.NoError(t, err)
120+
data, contentType, err := client.Download(ctx, resp.ID)
121+
require.NoError(t, err)
122+
123+
// then
124+
require.Equal(t, codersdk.ContentTypeTar, contentType)
125+
require.Equal(t, tarball, data)
126+
})
127+
128+
t.Run("InsertTar_DownloadZip", func(t *testing.T) {
129+
t.Parallel()
130+
client := coderdtest.New(t, nil)
131+
_ = coderdtest.CreateFirstUser(t, client)
132+
133+
// given
134+
tarball, err := echo.Tar(&echo.Responses{
135+
Parse: echo.ParseComplete,
136+
ProvisionApply: echo.ApplyComplete,
137+
})
138+
require.NoError(t, err)
139+
140+
tarReader := tar.NewReader(bytes.NewReader(tarball))
141+
expectedZip, err := coderd.CreateZipFromTar(tarReader)
142+
require.NoError(t, err)
143+
144+
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
145+
defer cancel()
146+
147+
// when
148+
resp, err := client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader(tarball))
149+
require.NoError(t, err)
150+
data, contentType, err := client.DownloadWithFormat(ctx, resp.ID, codersdk.FormatZip)
151+
require.NoError(t, err)
152+
153+
// then
154+
require.Equal(t, codersdk.ContentTypeZip, contentType)
155+
require.Equal(t, expectedZip, data)
156+
})
90157
}

‎coderd/fileszip.go‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
package coderd
2+
3+
import (
4+
"archive/tar"
5+
"archive/zip"
6+
"bytes"
7+
"errors"
8+
"io"
9+
"log"
10+
)
11+
12+
func CreateTarFromZip(zipReader *zip.Reader) ([]byte, error) {
13+
var tarBuffer bytes.Buffer
14+
err := writeTarArchive(&tarBuffer, zipReader)
15+
if err != nil {
16+
return nil, err
17+
}
18+
return tarBuffer.Bytes(), nil
19+
}
20+
21+
func writeTarArchive(w io.Writer, zipReader *zip.Reader) error {
22+
tarWriter := tar.NewWriter(w)
23+
defer tarWriter.Close()
24+
25+
for _, file := range zipReader.File {
26+
err := processFileInZipArchive(file, tarWriter)
27+
if err != nil {
28+
return err
29+
}
30+
}
31+
return nil
32+
}
33+
34+
func processFileInZipArchive(file *zip.File, tarWriter *tar.Writer) error {
35+
fileReader, err := file.Open()
36+
if err != nil {
37+
return err
38+
}
39+
defer fileReader.Close()
40+
41+
err = tarWriter.WriteHeader(&tar.Header{
42+
Name: file.Name,
43+
Size: file.FileInfo().Size(),
44+
Mode: 0o644,
45+
})
46+
if err != nil {
47+
return err
48+
}
49+
50+
n, err := io.CopyN(tarWriter, fileReader, httpFileMaxBytes)
51+
log.Println(file.Name, n, err)
52+
if errors.Is(err, io.EOF) {
53+
err = nil
54+
}
55+
return err
56+
}
57+
58+
func CreateZipFromTar(tarReader *tar.Reader) ([]byte, error) {
59+
var zipBuffer bytes.Buffer
60+
err := WriteZipArchive(&zipBuffer, tarReader)
61+
if err != nil {
62+
return nil, err
63+
}
64+
return zipBuffer.Bytes(), nil
65+
}
66+
67+
func WriteZipArchive(w io.Writer, tarReader *tar.Reader) error {
68+
zipWriter := zip.NewWriter(w)
69+
defer zipWriter.Close()
70+
71+
for {
72+
tarHeader, err := tarReader.Next()
73+
if errors.Is(err, io.EOF) {
74+
break
75+
}
76+
77+
if err != nil {
78+
return err
79+
}
80+
81+
zipHeader, err := zip.FileInfoHeader(tarHeader.FileInfo())
82+
if err != nil {
83+
return err
84+
}
85+
zipHeader.Name = tarHeader.Name
86+
87+
zipEntry, err := zipWriter.CreateHeader(zipHeader)
88+
if err != nil {
89+
return err
90+
}
91+
92+
_, err = io.CopyN(zipEntry, tarReader, httpFileMaxBytes)
93+
if errors.Is(err, io.EOF) {
94+
err = nil
95+
}
96+
if err != nil {
97+
return err
98+
}
99+
}
100+
return nil // don't need to flush as we call `writer.Close()`
101+
}

‎codersdk/files.go‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@ import (
1212

1313
const (
1414
ContentTypeTar = "application/x-tar"
15+
ContentTypeZip = "application/zip"
16+
17+
FormatZip = "zip"
1518
)
1619

1720
// UploadResponse contains the hash to reference the uploaded file.
@@ -38,7 +41,12 @@ func (c *Client) Upload(ctx context.Context, contentType string, rd io.Reader) (
3841

3942
// Download fetches a file by uploaded hash.
4043
func (c *Client) Download(ctx context.Context, id uuid.UUID) ([]byte, string, error) {
41-
res, err := c.Request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/files/%s", id.String()), nil)
44+
return c.DownloadWithFormat(ctx, id, "")
45+
}
46+
47+
// Download fetches a file by uploaded hash, but it forces format conversion.
48+
func (c *Client) DownloadWithFormat(ctx context.Context, id uuid.UUID, format string) ([]byte, string, error) {
49+
res, err := c.Request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/files/%s?format=%s", id.String(), format), nil)
4250
if err != nil {
4351
return nil, "", err
4452
}

‎docs/api/files.md‎

Lines changed: 5 additions & 5 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)