dev.phcode.dev/src/migrateAssist.html at gh-pages · phcode-dev/dev.phcode.dev · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
<!DOCTYPE html>
<!--
 * GNU AGPL-3.0 License
 *
 * Copyright (c) 2021 - present core.ai . All rights reserved.
 *
 * This program is free software: you can redistribute it and/or modify it
 * under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License
 * for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see https://opensource.org/licenses/AGPL-3.0.
-->
<html lang="en">
<head>
    <meta charset="utf-8">
    <title>Phoenix Code data migration helper</title>
</head>
<body>
<!--
    Migration helper, served from the origin that is being retired.

    web.phcode.dev embeds this page in a hidden iframe to read the projects, preferences and
    extensions that live in this origin's IndexedDB and cannot otherwise cross the origin boundary.
    This works because phcode.dev and web.phcode.dev are the same site (eTLD+1), so the frame is
    same-site rather than third party and browsers do not partition its storage.

    SECURITY: this page hands the user's entire browser filesystem to whoever embeds it, and the
    origin check below is the only thing standing in the way. Exact string matching only, on every
    single message, and replies always go to the validated origin, never "*". Reads are confined to
    the three roots in ALLOWED_ROOTS.

    It deliberately does not boot Phoenix. virtualfs.js is a self contained IIFE that gives us
    window.fs over the same Filer IndexedDB, which is all we need.

    Files are streamed one at a time rather than zipped per folder. IndexedDB costs roughly 10ms per
    file whatever we do, so an intermediate zip only adds a read/encode/decode pass on top of that
    floor, and measured on 300 files it produced an archive slightly LARGER than the input because
    JSZip stores uncompressed. Streaming also gives the receiver an exact per file progress count and
    keeps peak memory at one file instead of one whole folder.
-->
<script src="phoenix/virtualfs.js"></script>
<script>
    (function () {
        const TRUSTED_PARENT_ORIGINS = [
            "https://web.phcode.dev",
            "http://localhost:8000",
            "http://127.0.0.1:8000",
            "http://localhost:5000",
            "http://127.0.0.1:5000"
        ];

        const PROJECTS_ROOT = "/fs/local";
        const APP_ROOT = "/fs/app";
        const EXTENSIONS_USER = APP_ROOT + "/extensions/user";
        const EXTENSIONS_DISABLED = APP_ROOT + "/extensions/disabled";
        const PREFS_FILE = APP_ROOT + "/phcode.json";

        // Only these roots may ever be zipped and handed out. Anything else, including /mnt (which
        // holds real disc folders the user picked), is refused.
        const ALLOWED_ROOTS = [PROJECTS_ROOT, EXTENSIONS_USER, EXTENSIONS_DISABLED];

        // Auto created on both origins, so their presence proves nothing about the user having data.
        // They are still migrated when something else triggers the run, since they may be edited.
        const NOT_EVIDENCE_OF_DATA = ["default project", "explore"];

        // Extension folders carrying this marker are already tombstoned for deletion at next boot.
        const DELETED_EXTENSION_MARKER = "_phcode_extension_marked_for_delete";

        const PHSTORE_DB = "PhStore";
        const PHSTORE_STORE = "KVStore";
        const PHSTORE_KEYS = ["extensions.disabled", "STATE_recentProjects"];

        const CHUNK_SIZE = 8 * 1024 * 1024;

        let parentOrigin = null;

        function isTrustedOrigin(origin) {
            // Exact match only. A startsWith test here would let https://web.phcode.dev.evil.com
            // through, which would hand a stranger every file the user owns.
            return TRUSTED_PARENT_ORIGINS.indexOf(origin) !== -1;
        }

        function reply(message, transfer) {
            if (!parentOrigin) {
                return;
            }
            window.parent.postMessage(message, parentOrigin, transfer || []);
        }

        function fail(id, err) {
            console.error("migrateAssist: failed", id, err);
            reply({ type: "MIGRATE_ERROR", id: id, message: (err && err.message) || String(err) });
        }

        function readdir(dir) {
            return new Promise((resolve) => {
                fs.readdir(dir, { withFileTypes: true }, (err, entries) => {
                    resolve(err ? [] : (entries || []));
                });
            });
        }

        function readFileBytes(filePath) {
            return new Promise((resolve, reject) => {
                fs.readFile(filePath, fs.BYTE_ARRAY_ENCODING, (err, data) => {
                    if (err) {
                        reject(err);
                        return;
                    }
                    resolve(data);
                });
            });
        }

        function readFileText(filePath) {
            return new Promise((resolve) => {
                fs.readFile(filePath, "utf8", (err, data) => {
                    resolve(err ? null : data);
                });
            });
        }

        /**
         * Walks a directory and returns every file in it as {path, relativePath, size}.
         */
        async function walk(rootDir, currentDir, out) {
            const entries = await readdir(currentDir);
            for (const entry of entries) {
                const fullPath = currentDir + "/" + entry.name;
                if (entry.isDirectory()) {
                    await walk(rootDir, fullPath, out);
                } else {
                    out.push({
                        path: fullPath,
                        relativePath: fullPath.substring(rootDir.length + 1),
                        size: entry.size || 0
                    });
                }
            }
            return out;
        }

        function readPhStore() {
            return new Promise((resolve) => {
                const values = {};
                let request;
                try {
                    request = indexedDB.open(PHSTORE_DB);
                } catch (e) {
                    resolve(values);
                    return;
                }
                request.onerror = () => resolve(values);
                request.onsuccess = () => {
                    const db = request.result;
                    if (!db.objectStoreNames.contains(PHSTORE_STORE)) {
                        db.close();
                        resolve(values);
                        return;
                    }
                    const store = db.transaction(PHSTORE_STORE, "readonly").objectStore(PHSTORE_STORE);
                    let pending = PHSTORE_KEYS.length;
                    const done = () => {
                        pending = pending - 1;
                        if (pending === 0) {
                            db.close();
                            resolve(values);
                        }
                    };
                    for (const key of PHSTORE_KEYS) {
                        const get = store.get(key);
                        get.onsuccess = () => {
                            // PhStore wraps every value as {t: mtime, v: jsonString}.
                            const entry = get.result;
                            if (entry && typeof entry.v === "string") {
                                try {
                                    values[key] = JSON.parse(entry.v);
                                } catch (e) {
                                    console.warn("migrateAssist: unparsable PhStore value", key);
                                }
                            }
                            done();
                        };
                        get.onerror = done;
                    }
                };
                // A brand new database created by the open above has no store and no data, which the
                // contains() check above already handles.
                request.onupgradeneeded = () => { /* nothing to do, we only ever read */ };
            });
        }

        async function collectFiles() {
            const files = [];
            let hasRealProject = false;

            const projectDirs = await readdir(PROJECTS_ROOT);
            for (const entry of projectDirs) {
                if (!entry.isDirectory()) {
                    continue;
                }
                const dir = PROJECTS_ROOT + "/" + entry.name;
                const found = await walk(dir, dir, []);
                if (!found.length) {
                    continue;
                }
                for (const f of found) {
                    files.push({ path: f.path, size: f.size });
                }
                if (NOT_EVIDENCE_OF_DATA.indexOf(entry.name) === -1) {
                    hasRealProject = true;
                }
            }

            let extensionCount = 0;
            for (const extRoot of [EXTENSIONS_USER, EXTENSIONS_DISABLED]) {
                const extDirs = await readdir(extRoot);
                const keep = [];
                for (const entry of extDirs) {
                    if (!entry.isDirectory()) {
                        continue;
                    }
                    const markerPath = extRoot + "/" + entry.name + "/" + DELETED_EXTENSION_MARKER;
                    if (await readFileText(markerPath) !== null) {
                        continue; // already tombstoned for deletion, do not carry it over
                    }
                    keep.push(entry.name);
                }
                if (!keep.length) {
                    continue;
                }
                if (extRoot === EXTENSIONS_USER) {
                    extensionCount = keep.length;
                }
                const found = await walk(extRoot, extRoot, []);
                for (const f of found) {
                    if (keep.indexOf(f.relativePath.split("/")[0]) === -1) {
                        continue;
                    }
                    if (f.relativePath.endsWith(DELETED_EXTENSION_MARKER)) {
                        continue;
                    }
                    files.push({ path: f.path, size: f.size });
                }
            }

            const prefsText = await readFileText(PREFS_FILE);
            let hasPrefs = false;
            if (prefsText) {
                try {
                    const parsed = JSON.parse(prefsText);
                    hasPrefs = !!parsed && Object.keys(parsed).length > 0;
                } catch (e) {
                    hasPrefs = false;
                }
            }

            return {
                files: files,
                prefs: hasPrefs ? prefsText : null,
                hasData: hasRealProject || hasPrefs || extensionCount > 0
            };
        }

        async function handleScan() {
            const scan = await collectFiles();
            const phStore = await readPhStore();
            reply({
                type: "MIGRATE_SCAN_RESULT",
                hasData: scan.hasData,
                files: scan.files,
                totalBytes: scan.files.reduce((sum, f) => sum + f.size, 0),
                prefs: scan.prefs,
                phStore: phStore
            });
        }

        function isAllowedFile(filePath) {
            if (typeof filePath !== "string" || filePath.indexOf("..") !== -1) {
                return false;
            }
            // Must sit under one of the three roots we are willing to hand over. Anything else,
            // /mnt included, is refused however it is spelled.
            return ALLOWED_ROOTS.some((root) => filePath.startsWith(root + "/"));
        }

        async function handleRead(path, offset, length) {
            if (!isAllowedFile(path)) {
                fail(path, new Error("Refused: path is outside the migratable roots"));
                return;
            }
            const bytes = await readFileBytes(path);
            const start = typeof offset === "number" ? offset : 0;
            const size = typeof length === "number" ? length : CHUNK_SIZE;
            const end = Math.min(start + size, bytes.byteLength);
            const chunk = bytes.slice(start, end);
            reply({
                type: "MIGRATE_DATA",
                path: path,
                offset: start,
                chunk: chunk,
                eof: end >= bytes.byteLength
            }, [chunk]);
        }

        window.addEventListener("message", function (event) {
            // Re-checked on every message, not just the handshake.
            if (!isTrustedOrigin(event.origin) || event.origin !== parentOrigin) {
                return;
            }
            const data = event.data;
            if (!data || typeof data !== "object") {
                return;
            }
            if (data.type === "MIGRATE_SCAN") {
                handleScan().catch((err) => fail("scan", err));
            } else if (data.type === "MIGRATE_READ") {
                handleRead(data.path, data.offset, data.length).catch((err) => fail(data.path, err));
            }
        });

        const requestedParent = new URLSearchParams(window.location.search || "").get("parentOrigin");
        if (isTrustedOrigin(requestedParent)) {
            parentOrigin = requestedParent;
            reply({ type: "MIGRATE_READY" });
        } else {
            // Stay silent. Never post to an unvalidated origin, not even to say no.
            console.error("migrateAssist: refusing to run, untrusted parent origin", requestedParent);
        }
    }());
</script>
</body>
</html>