insert into db script fixes

db/update_db.js

@@ -22,52 +22,59 @@ const lookup = {
     tracks: 'tracks'
 };
 
-// Retrieve existing IDs from Stream
-const existingIds = db
-    .prepare('SELECT id FROM Stream')
-    .all()
-    .map((row) => row.id);
-// Create a set of existing IDs for efficient lookup
-const idSet = new Set(existingIds);
+const existingRow = db.prepare('SELECT * FROM Stream WHERE id = ?');
 
-// Check if --overwrite flag is set
+// Overwrite differing streams without asking
 const shouldOverwrite = process.argv.includes('--overwrite');
 
 // Process JSON files and insert data into Stream
-fs.readdir(jsonFolder, (err, files) => {
-    if (err) throw err;
-    console.log(`Found ${files.length} JSON file(s) in ${jsonFolder}.`);
+const files = fs.readdirSync(jsonFolder);
+console.log(`Found ${files.length} JSON file(s) in ${jsonFolder}.`);
 
-    files.forEach((file) => {
-        if (!file.endsWith('.json')) return;
-        if (file.endsWith('.sideload.json')) return;
+for (const file of files) {
+    if (!file.endsWith('.json')) continue;
+    if (file.endsWith('.sideload.json')) continue;
 
-        const jsonString = fs.readFileSync(path.join(jsonFolder, file), 'utf8');
-        let jsonData = JSON.parse(jsonString);
+    const jsonString = fs.readFileSync(path.join(jsonFolder, file), 'utf8');
+    let jsonData = JSON.parse(jsonString);
 
-        const sideloadPath = path.join(jsonFolder, file.slice(0, -5) + '.sideload.json');
-        if (fs.existsSync(sideloadPath, 'utf8')) {
-            const sideloadData = JSON.parse(fs.readFileSync(sideloadPath));
-            jsonData = { ...jsonData, ...sideloadData };
-        }
-        // Skip if ID already exists in Stream
-        if (idSet.has(jsonData.id)) {
-            if (!shouldOverwrite) {
-                console.log(`Skipped data for ID ${jsonData.id} (already exists).`);
-                return;
-            }
-            console.log(`Overwriting data for ID ${jsonData.id}.`);
+    const sideloadPath = path.join(jsonFolder, file.slice(0, -5) + '.sideload.json');
+    if (fs.existsSync(sideloadPath)) {
+        const sideloadData = JSON.parse(fs.readFileSync(sideloadPath));
+        jsonData = { ...jsonData, ...sideloadData };
+    }
+
+    // A key with no column would otherwise reach the query as `undefined`
+    const unknown = Object.keys(jsonData).filter((key) => !(key in lookup));
+    if (unknown.length > 0)
+        console.log(`Ignored unknown key(s) in ${file}: ${unknown.join(', ')}.`);
+
+    // Prepare attributes for insertion
+    const fields = Object.entries(jsonData)
+        .filter(([key]) => key in lookup)
+        // Serialize value if array
+        .map(([key, value]) => [lookup[key], Array.isArray(value) ? JSON.stringify(value) : value]);
+
+    const row = existingRow.get(jsonData.id);
+    if (row) {
+        const changed = fields
+            .filter(([column, value]) => String(row[column] ?? '') !== String(value ?? ''))
+            .map(([column]) => column);
+
+        if (changed.length === 0) {
+            console.log(`Unchanged data for ID ${jsonData.id}.`);
+            continue;
         }
 
-        // Prepare attributes for insertion
-        const values = Object.keys(jsonData).map(
-            // Serialize value if array
-            (key) => (Array.isArray(jsonData[key]) ? JSON.stringify(jsonData[key]) : jsonData[key])
-        );
-        const columns = Object.keys(jsonData).map((key) => lookup[key]);
+        console.log(`ID ${jsonData.id} differs from the database in: ${changed.join(', ')}.`);
+        if (!shouldOverwrite && !confirm(`Overwrite ID ${jsonData.id}?`)) {
+            console.log(`Kept database version of ID ${jsonData.id}.`);
+            continue;
+        }
+    }
 
-        const sql = `INSERT OR REPLACE INTO Stream (${columns.join(', ')}) VALUES (${columns.map(() => '?').join(', ')})`;
-        db.prepare(sql).run(...values);
-        console.log(`Inserted data for ID ${jsonData.id}.`);
-    });
-});
+    const columns = fields.map(([column]) => column);
+    const sql = `INSERT OR REPLACE INTO Stream (${columns.join(', ')}) VALUES (${columns.map(() => '?').join(', ')})`;
+    db.prepare(sql).run(...fields.map(([, value]) => value));
+    console.log(`Inserted data for ID ${jsonData.id}.`);
+}