Lost in the middle of the script: 38k records halted in their tracks. How to pick up the pieces and resume the DynamoDB journey from where it paused?

When inserting a large amount of data into DynamoDB, it’s common to encounter issues such as timeouts or interruptions, especially if the process takes a significant amount of time.

One way to achieve this is by keeping track of the progress of the insertion process, typically by storing the index of the last successfully inserted record. Here’s how you can modify your script to implement this:

const AWS = require('aws-sdk');
const fs = require('fs');

AWS.config.update({ region: 'ap-south-1' });

const docClient = new AWS.DynamoDB.DocumentClient();
const rawData = fs.readFileSync('movies.json');
const moviesData = JSON.parse(rawData);

const checkpointFileName = 'checkpoint.json';

async function insertData() {
    let lastProcessedIndex = 0;

    try {
        // Load checkpoint if it exists
        if (fs.existsSync(checkpointFileName)) {
            const checkpointData = fs.readFileSync(checkpointFileName, 'utf8');
            lastProcessedIndex = JSON.parse(checkpointData).lastProcessedIndex || 0;
        }

        for (let i = lastProcessedIndex; i < moviesData.length; i++) {
            const movie = moviesData[i];

            // Insert movie data into the Movies table
            const movieParams = {
                TableName: 'Movies', // Specify the table name
                Item: {
                    movieId: movie.title.replace(/\s+/g, '_'), // Generate movieId from the movie title
                    title: movie.title,
                    year: movie.year,
                    genres: movie.genres,
                    href: movie.href,
                    extract: movie.extract,
                    thumbnail: movie.thumbnail,
                    thumbnail_width: movie.thumbnail_width,
                    thumbnail_height: movie.thumbnail_height
                }
            };
            await docClient.put(movieParams).promise();

            // Insert cast members into CastMembers table and create entries in MovieCast table
            for (const castMember of movie.cast) {
                // Insert cast member data into CastMembers table
                const castId = `${movie.title.replace(/\s+/g, '_')}_${castMember.replace(/\s+/g, '_')}`;
                const castParams = {
                    TableName: 'CastMembers', // Specify the table name
                    Item: {
                        castId: castId,
                        name: castMember
                    }
                };
                await docClient.put(castParams).promise();

                // Create entry in MovieCast table
                const movieCastParams = {
                    TableName: 'MovieCast', // Specify the table name
                    Item: {
                        movieId: movie.title.replace(/\s+/g, '_'),
                        castId: castId
                    }
                };
                await docClient.put(movieCastParams).promise();
            }

            console.log(`Inserted data for movie: ${movie.title}`);

            // Update checkpoint after processing each movie
            lastProcessedIndex = i + 1;
            fs.writeFileSync(checkpointFileName, JSON.stringify({ lastProcessedIndex }));
        }

        console.log('All data inserted successfully');
        // Remove checkpoint file after successful completion
        fs.unlinkSync(checkpointFileName);
    } catch (error) {
        console.error('Error inserting data:', error);
        // You may choose to handle errors differently, e.g., retrying or logging failed items
    }
}

insertData();