Skip to content

Commit 9083434

Browse files
rohansinghrohansingh
authored
bundle: Add WithoutRuntime write option (#1051)
WithoutRuntime is a WriteOption that can be used to write the bundle without using the runtime to determine the files to include in the bundle. Instead, all files in the source FS will be included in the bundle. This is useful when writing a bundle that is known not to contain any unnecessary files, when loading and rewriting a bundle that was already tree-shaken, or when loading the entire runtime is not possible for performance or security reasons.
1 parent 0ad59b9 commit 9083434

File tree

3 files changed

+181
-93
lines changed

3 files changed

+181
-93
lines changed

‎bundle/bundle.go‎

Lines changed: 0 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,16 @@
22
package bundle
33

44
import (
5-
"archive/tar"
65
"bytes"
76
"compress/gzip"
87
"fmt"
98
"io"
109
"io/fs"
1110
"os"
12-
"path/filepath"
1311

1412
"github.com/nlepage/go-tarfs"
1513

1614
"tidbyt.dev/pixlet/manifest"
17-
"tidbyt.dev/pixlet/runtime"
1815
)
1916

2017
const (
@@ -80,93 +77,3 @@ func LoadBundle(in io.Reader) (*AppBundle, error) {
8077

8178
return FromFS(fs)
8279
}
83-
84-
// WriteBundleToPath is a helper to be able to write the bundle to a provided
85-
// directory.
86-
func (b *AppBundle) WriteBundleToPath(dir string) error {
87-
path := filepath.Join(dir, AppBundleName)
88-
f, err := os.Create(path)
89-
if err != nil {
90-
return fmt.Errorf("could not create file for bundle: %w", err)
91-
}
92-
defer f.Close()
93-
94-
return b.WriteBundle(f)
95-
}
96-
97-
// WriteBundle writes a compressed archive to the provided writer.
98-
func (ab *AppBundle) WriteBundle(out io.Writer) error {
99-
// we don't want to naively write the entire source FS to the tarball,
100-
// since it could contain a lot of extraneous files. instead, run the
101-
// applet and interrogate it for the files it needs to include in the
102-
// bundle.
103-
app, err := runtime.NewAppletFromFS(ab.Manifest.ID, ab.Source, runtime.WithPrintDisabled())
104-
if err != nil {
105-
return fmt.Errorf("loading applet for bundling: %w", err)
106-
}
107-
bundleFiles := app.PathsForBundle()
108-
109-
// Setup writers.
110-
gzw := gzip.NewWriter(out)
111-
defer gzw.Close()
112-
113-
tw := tar.NewWriter(gzw)
114-
defer tw.Close()
115-
116-
// Write manifest.
117-
buff := &bytes.Buffer{}
118-
err = ab.Manifest.WriteManifest(buff)
119-
if err != nil {
120-
return fmt.Errorf("could not write manifest to buffer: %w", err)
121-
}
122-
b := buff.Bytes()
123-
124-
hdr := &tar.Header{
125-
Name: manifest.ManifestFileName,
126-
Mode: 0600,
127-
Size: int64(len(b)),
128-
}
129-
err = tw.WriteHeader(hdr)
130-
if err != nil {
131-
return fmt.Errorf("could not write manifest header: %w", err)
132-
}
133-
_, err = tw.Write(b)
134-
if err != nil {
135-
return fmt.Errorf("could not write manifest to archive: %w", err)
136-
}
137-
138-
// write sources.
139-
for _, path := range bundleFiles {
140-
stat, err := fs.Stat(ab.Source, path)
141-
if err != nil {
142-
return fmt.Errorf("could not stat %s: %w", path, err)
143-
}
144-
145-
hdr, err := tar.FileInfoHeader(stat, "")
146-
if err != nil {
147-
return fmt.Errorf("creating header for %s: %w", path, err)
148-
}
149-
hdr.Name = filepath.ToSlash(path)
150-
151-
err = tw.WriteHeader(hdr)
152-
if err != nil {
153-
return fmt.Errorf("writing header for %s: %w", path, err)
154-
}
155-
156-
if !stat.IsDir() {
157-
file, err := ab.Source.Open(path)
158-
if err != nil {
159-
return fmt.Errorf("opening file %s: %w", path, err)
160-
}
161-
162-
written, err := io.Copy(tw, file)
163-
if err != nil {
164-
return fmt.Errorf("writing file %s: %w", path, err)
165-
} else if written != stat.Size() {
166-
return fmt.Errorf("did not write entire file %s: %w", path, err)
167-
}
168-
}
169-
}
170-
171-
return nil
172-
}

‎bundle/bundle_test.go‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,45 @@ func TestBundleWriteAndLoad(t *testing.T) {
5050
_, err = newBun.Source.Open("unused.txt")
5151
assert.ErrorIs(t, err, os.ErrNotExist)
5252
}
53+
54+
func TestBundleWriteAndLoadWithoutRuntime(t *testing.T) {
55+
ab, err := bundle.FromDir("testdata/testapp")
56+
assert.NoError(t, err)
57+
assert.Equal(t, "test-app", ab.Manifest.ID)
58+
assert.NotNil(t, ab.Source)
59+
60+
// Create a temp directory.
61+
dir, err := os.MkdirTemp("", "")
62+
assert.NoError(t, err)
63+
64+
// Write bundle to the temp directory, without tree-shaking.
65+
err = ab.WriteBundleToPath(dir, bundle.WithoutRuntime())
66+
assert.NoError(t, err)
67+
68+
// Ensure we can load up the bundle just created.
69+
path := filepath.Join(dir, bundle.AppBundleName)
70+
f, err := os.Open(path)
71+
assert.NoError(t, err)
72+
defer f.Close()
73+
newBun, err := bundle.LoadBundle(f)
74+
assert.NoError(t, err)
75+
assert.Equal(t, "test-app", newBun.Manifest.ID)
76+
assert.NotNil(t, ab.Source)
77+
78+
// Ensure the loaded bundle contains the files we expect.
79+
filesExpected := []string{
80+
"manifest.yaml",
81+
"test_app.star",
82+
"test.txt",
83+
"a_subdirectory/hi.jpg",
84+
"unused.txt",
85+
}
86+
for _, file := range filesExpected {
87+
_, err := newBun.Source.Open(file)
88+
assert.NoError(t, err)
89+
}
90+
}
91+
5392
func TestLoadBundle(t *testing.T) {
5493
f, err := os.Open("testdata/bundle.tar.gz")
5594
assert.NoError(t, err)

‎bundle/write.go‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
// Package bundle provides primitives for bundling apps for portability.
2+
package bundle
3+
4+
import (
5+
"archive/tar"
6+
"bytes"
7+
"compress/gzip"
8+
"fmt"
9+
"io"
10+
"io/fs"
11+
"os"
12+
"path/filepath"
13+
"slices"
14+
15+
"tidbyt.dev/pixlet/manifest"
16+
"tidbyt.dev/pixlet/runtime"
17+
)
18+
19+
type WriteOption interface{}
20+
21+
type withoutRuntimeOption struct{}
22+
23+
// WithoutRuntime is a WriteOption that can be used to write the bundle without
24+
// using the runtime to determine the files to include in the bundle. Instead,
25+
// all files in the source FS will be included in the bundle.
26+
//
27+
// This is useful when writing a bundle that is known not to contain any
28+
// unnecessary files, when loading and rewriting a bundle that was already
29+
// tree-shaken, or when loading the entire runtime is not possible for
30+
// performance or security reasons.
31+
func WithoutRuntime() WriteOption {
32+
return withoutRuntimeOption{}
33+
}
34+
35+
// WriteBundleToPath is a helper to be able to write the bundle to a provided
36+
// directory.
37+
func (b *AppBundle) WriteBundleToPath(dir string, opts ...WriteOption) error {
38+
path := filepath.Join(dir, AppBundleName)
39+
f, err := os.Create(path)
40+
if err != nil {
41+
return fmt.Errorf("could not create file for bundle: %w", err)
42+
}
43+
defer f.Close()
44+
45+
return b.WriteBundle(f, opts...)
46+
}
47+
48+
// WriteBundle writes a compressed archive to the provided writer.
49+
func (ab *AppBundle) WriteBundle(out io.Writer, opts ...WriteOption) error {
50+
var bundleFiles []string
51+
52+
if slices.Contains(opts, WithoutRuntime()) {
53+
// we can't use the runtime to determine the files to include in the
54+
// bundle, so we'll just include everything in the source FS.
55+
err := fs.WalkDir(ab.Source, ".", func(path string, d fs.DirEntry, err error) error {
56+
if err != nil {
57+
return fmt.Errorf("walking directory: %w", err)
58+
}
59+
if !d.IsDir() {
60+
bundleFiles = append(bundleFiles, path)
61+
}
62+
return nil
63+
})
64+
if err != nil {
65+
return fmt.Errorf("walking source FS: %w", err)
66+
}
67+
} else {
68+
// we don't want to naively write the entire source FS to the tarball,
69+
// since it could contain a lot of extraneous files. instead, run the
70+
// applet and interrogate it for the files it needs to include in the
71+
// bundle.
72+
app, err := runtime.NewAppletFromFS(ab.Manifest.ID, ab.Source, runtime.WithPrintDisabled())
73+
if err != nil {
74+
return fmt.Errorf("loading applet for bundling: %w", err)
75+
}
76+
bundleFiles = app.PathsForBundle()
77+
}
78+
79+
// Setup writers.
80+
gzw := gzip.NewWriter(out)
81+
defer gzw.Close()
82+
83+
tw := tar.NewWriter(gzw)
84+
defer tw.Close()
85+
86+
// Write manifest.
87+
buff := &bytes.Buffer{}
88+
err := ab.Manifest.WriteManifest(buff)
89+
if err != nil {
90+
return fmt.Errorf("could not write manifest to buffer: %w", err)
91+
}
92+
b := buff.Bytes()
93+
94+
hdr := &tar.Header{
95+
Name: manifest.ManifestFileName,
96+
Mode: 0600,
97+
Size: int64(len(b)),
98+
}
99+
err = tw.WriteHeader(hdr)
100+
if err != nil {
101+
return fmt.Errorf("could not write manifest header: %w", err)
102+
}
103+
_, err = tw.Write(b)
104+
if err != nil {
105+
return fmt.Errorf("could not write manifest to archive: %w", err)
106+
}
107+
108+
// write sources.
109+
for _, path := range bundleFiles {
110+
stat, err := fs.Stat(ab.Source, path)
111+
if err != nil {
112+
return fmt.Errorf("could not stat %s: %w", path, err)
113+
}
114+
115+
hdr, err := tar.FileInfoHeader(stat, "")
116+
if err != nil {
117+
return fmt.Errorf("creating header for %s: %w", path, err)
118+
}
119+
hdr.Name = filepath.ToSlash(path)
120+
121+
err = tw.WriteHeader(hdr)
122+
if err != nil {
123+
return fmt.Errorf("writing header for %s: %w", path, err)
124+
}
125+
126+
if !stat.IsDir() {
127+
file, err := ab.Source.Open(path)
128+
if err != nil {
129+
return fmt.Errorf("opening file %s: %w", path, err)
130+
}
131+
132+
written, err := io.Copy(tw, file)
133+
if err != nil {
134+
return fmt.Errorf("writing file %s: %w", path, err)
135+
} else if written != stat.Size() {
136+
return fmt.Errorf("did not write entire file %s: %w", path, err)
137+
}
138+
}
139+
}
140+
141+
return nil
142+
}

0 commit comments

Comments
 (0)