AMAZON DynamoDB(3)CRUD Item

AMAZON DynamoDB(3)CRUD Item

BatchWriteItem we are using that to save data into DynamoDB. But BatchWriteItem cannot update items, to update items, use the UpdateItem action.

DynamoDB Operation List
https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Operations.html

How to Truncate all data in one table
https://gist.github.com/k3karthic/4bc929885eef40dbe010

We can delete the unused columns if we need to
delete item.serialNumber1;

Here is some code to do that thing

"use strict";

// How to run
// eg: node dynamodb-scripts/import-devices-to-dynamodb.js int ./css_devices_only_csv.txt

var importDevicesToDynamo;
(function (importDevicesToDynamo) {
    const fs = require('fs');
    const babyparse = require("babyparse");
    const AWS = require("aws-sdk");
    const log4js = require('log4js');
    const logger = log4js.getLogger();
    const sleep = require('sleep');

    const DEV_DYNAMODB_URL = "http://dynamodb.us-west-1.amazonaws.com";
    const DEV_DYNAMO_ACCESS_KEY = "xxxxx";
    const DEV_DYNAMO_SECRET_KEY = "xxxxxx/xxx";
    const DEV_DYNAMO_REGION = "us-west-1";

    const STAGE_DYNAMODB_URL = "http://dynamodb.us-west-1.amazonaws.com";
    const STAGE_DYNAMO_ACCESS_KEY = "xxxxxxx";
    const STAGE_DYNAMO_SECRET_KEY = "xxxxx/xxxxxxx";
    const STAGE_DYNAMO_REGION = "us-west-1";

    const PROD_DYNAMODB_URL = "http://dynamodb.us-west-2.amazonaws.com";
    const PROD_DYNAMO_ACCESS_KEY = "xxxxxx";
    const PROD_DYNAMO_SECRET_KEY = "xxxxxxxxxxxx";
    const PROD_DYNAMO_REGION = "us-west-2";

    const env = process.argv[2]; // Must be int, stage or prod
    const csvFilePath = process.argv[3];
   
    const config = {
        delimiter: ',',
        newline: "",
        quoteChar: '"',
        header: true,
        dynamicTyping: false,
        preview: 0,
        encoding: "utf8",
        worker: false,
        comments: false,
        skipEmptyLines: true
    };

    let tableName = `sillycat_device-${env}-devicePairing`;
    let accessKey = "";
    let signatureKey = "";
    let region = "";
    let dynamoDbUrl = "";
    //validate parameters
    if (!env) {
        console.log("\nMust pass in environment for 1st argument. Must be one of 'int, 'stage' or 'prod'");
        console.log("\nUsage - node dynamodb-scripts/import-devices-to-dynamodb.js {env} {csv path/file } ");
        console.log("\nExample - node dynamodb-scripts/import-devices-to-dynamodb.js int ./css_devices_only_csv.txt \n");
        process.exit(1);
    }
    if (!csvFilePath) {
        console.log("\nMust pass in csvFilePath for 2nd argument.");
        console.log("\nUsage - node dynamodb-scripts/import-devices-to-dynamodb.js {env} {csv path/file } ");
        console.log("\nExample - node dynamodb-scripts/import-devices-to-dynamodb.js int ./css_devices_only_csv.txt \n");
        process.exit(2);
    }
    if (env.toLowerCase() === 'int' || env.toLowerCase() === 'test') {
        dynamoDbUrl = DEV_DYNAMODB_URL;
        accessKey = DEV_DYNAMO_ACCESS_KEY;
        signatureKey = DEV_DYNAMO_SECRET_KEY;
        region = DEV_DYNAMO_REGION;
    } else if (env.toLowerCase() === 'stage') {
        dynamoDbUrl = STAGE_DYNAMODB_URL;
        accessKey = STAGE_DYNAMO_ACCESS_KEY;
        signatureKey = STAGE_DYNAMO_SECRET_KEY;
        region = STAGE_DYNAMO_REGION;
    } else if (env.toLowerCase() === 'prod') {
        dynamoDbUrl = PROD_DYNAMODB_URL;
        accessKey = PROD_DYNAMO_ACCESS_KEY;
        signatureKey = PROD_DYNAMO_SECRET_KEY;
        region = PROD_DYNAMO_REGION;
    } else {
        console.log("Must pass in environment for 1st argument. Must be one of 'int, 'stage' or 'prod'");
        process.exit(1);
    }

    console.log("Env = " + env);
    console.log("File to import = " + csvFilePath);
    let content = fs.readFileSync(csvFilePath, config);
    let parsed = babyparse.parse(content, config);
    let rows = JSON.parse(JSON.stringify(parsed.data));
    console.log("Row count = " + Object.keys(rows).length);
    let _id;
    // For the batch size of 10, we need to temporarily change the write capacity units to 50 in DynaoDB for the appropriate table, then reset to default when script is finished
    let size = 10;
    console.log("dynamoDbURL = " + dynamoDbUrl);
    console.log("tableName = " + tableName);

    AWS.config.update({
        accessKeyId: accessKey,
        secretAccessKey: signatureKey,
        region: region,
        endpoint: dynamoDbUrl
    });

    const dynamoDb = new AWS.DynamoDB.DocumentClient();
    for (let i = 0; i < rows.length; i += size) {
        // Slice the array into smaller arrays of 10,000 items
        let smallarray = rows.slice(i, i + size);
        console.log("i = " + i + " serialNumber = " + smallarray[0].serialNumber);
        let batchItems = smallarray.map(function (item) {
            try {
                // Replace empty string values with null. DynamoDB doesn't allow empty strings, will throw error on request.
                for (let items in item) {
                    let value = item[items];
                    if (value === undefined || value === "") {
                        item[items] = null;
                    }

                    if (items == "enabled") {
                        if (value === "f") {
                            item[items] = false;
                        } else if (value === "t") {
                            item[items] = true;
                        }
                    }
                }
                delete item.serialNumber1;

                let params = {
                    PutRequest: { Item: JSON.parse(JSON.stringify(item)) }
                };
                return params;
            }
            catch (error) {
                console.log("**** ERROR processing file: " + error);
            }
        });
        let batchRequestParams = '{"RequestItems":{"' + tableName + '":' + JSON.stringify(batchItems) + '},"ReturnConsumedCapacity":"TOTAL","ReturnItemCollectionMetrics": "SIZE"}';
        console.log("batchRequestParams = " + batchRequestParams);
        callDynamo(batchRequestParams).then(function (data) {
            sleep.msleep(100);
        })
            .catch(console.error);
    }
    function callDynamo(batchRequestParams) {
        return new Promise(function (resolve, reject) {
            dynamoDb.batchWrite(JSON.parse(batchRequestParams), function (err, data) {
                try {
                    if (err) {
                        logger.error(`Error - ${err} = Trying again:`);
                        sleep.msleep(100);
                        dynamoDb.batchWrite(JSON.parse(batchRequestParams), function (err, data) {
                            try {
                                if (err) {
                                    logger.error("Unable to add item a 2nd time, Error:", err);
                                    return reject(err);
                                }
                                else {
                                    logger.debug("2nd PutItem succeeded");
                                    resolve(data);
                                }
                            }
                            catch (error) {
                                console.log("error calling DynamoDB - " + error);
                                return reject(err);
                            }
                        });
                    }
                    else {
                        logger.debug("PutItem succeeded");
                        resolve(data);
                    }
                }
                catch (error) {
                    console.log("error calling DynamoDB - " + error);
                    return reject(err);
                }
            });
        });
    }
})(importDevicesToDynamo || (importDevicesToDynamo = {}));



References:
https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GettingStarted.NodeJs.03.html
https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchWriteItem.html