apc-pipeline-component

Library provides pipeline element infrastructure for a cradle data pipeline

Motivation

The pipeline component designed to provide a RabbitMQ subscriber and publisher ecosystem around a transform() function and should simplify implementation of future Cradle components.

Installation

npm i --save apc-pipeline-component

Usage

// my-transform.ts
import { Action, IMessage, IResult } from 'apc-pipeline-component

export default function(input: IMessage): IResult {
    const counter = input.data.counter;   // Simple transform function that increments a counter

    return {
        action: Action.Forward,
        target: '',     // can specify an exchange topic name
        message: {
            kind: 'myKind',
            data: {
                counter: counter + 1
            }
        }
    };
}

// start.ts
/**
 * Sub command to start a pipeline element with my-transform function
 */
import configProvider from '@vamship/config';
import loggerProvider from '@vamship/logger';
import { PipelineElement } from 'apc-pipeline-component
import transform from '../lib/my-transform';

export const command = 'start';
export const describe = 'Run Pipeline Transform';
export const builder = {};
export const handler = (argv) => {
    const config = configProvider.getConfig();
    const logger = loggerProvider.getLogger('command:start');
    logger.trace('config', { config });

    const pipelineElement = new PipelineElement(transform, config);

    return pipelineElement.start();
};

// .myProjectNamerc
{
  "default": {
    "app": {},
    "log": {
        "level": "trace",
        "extremeLogging": false
    },
    "subscriber": {
      "hostname": "localhost",
      "port": "5672",
      "username": "user",
      "password": "password",
      "exchangeName": "inputExchange",
      "exchangeType": "topic",
      "isExchangeDurable": "true",
      "topicName": "myTopic",
      "queueName": "inputQueue",
      "isQueueDurable": "true",
      "isQueueExclusive": "false"
    },
    "publisher": {
      "hostname": "localhost",
      "port": "5672",
      "username": "user",
      "password": "password",
      "exchangeName": "outputExchange",
      "exchangeType": "topic",
      "isExchangeDurable": "true",
      "queueName": "outputQueue",
      "isQueueDurable": "true",
      "isQueueExclusive": "false"
    }
  },
  "development": {
    "app": {}
  },
  "test": {
    "app": {}
  },
  "production": {
    "app": { },
    "log": {
        "level": "info",
        "extremeLogging": true
    }
  }
}

Testing

Use Docker to spin up a RabbitMQ container: docker run -d -p 5672:5672 -p 15672:15672 --rm --hostname my-rabbit --name some-rabbit rabbitmq:3-management

Launch your CLI application: ./working/src/bin/my-project.js start 2>&1

Open RabbitMQ management console in a web browser: http://localhost:15672 User: guest. Password: guest.

Use the management console to send a message input the input exchange and to receive the message on the output exchange.

Tip: you can find the rabbit connection again using the

docker ps

Contributing

To ensure that your code works fine you will need to test it against running RabbitMQ instance.

NOTE: Due to an unresolved issue with the "gulp-eslint" plugin, the gulp lint task is currently broken. Refer to IDE or some other tooling for linting this project.

API Documentation

API documentation can be found [here](https://git-codecommit.us-east-1.amazonaws.com/v1/repos/apc-pipeline-component