Fork me on GitHub

indian-ocean

Build Status NPM version NPM downloads js-standard-style

A NodeJS library for reading and writing data and handy file system utilities. Supports csv, tsv, json, yaml, dbf as well as custom formats and delimeters.

npm install indian-ocean

Example:

var io = require('indian-ocean')

var json_data = io.readDataSync('path/to/data.csv')

console.log(json_data)

/*
[
  {
    "name": "Gerald",
    "occupation": "Teacher",
    "city": "Philadelphia"
  },
  {
    "name": "Marcy",
    "occupation": "Venture Capitalist",
    "city": "New York"
  }
]
*/

io.writeDataSync('path/to/save/output.json', json_data)
              

API Documentation

converters

Functions that both read and write.

convertDbfToData(inFileName, outFileName, [options], callback)

Reads in a dbf file with .readDbf and write to file using .writeData. A convenience function for converting DBFs to more useable formats. Formerly known as writeDbfToData and is aliased for legacy support.

parameter type description
inFileName String

the input file name

outFileName String

the output file name

options [Object]

Optional config object, see below

options.makeDirectories [Boolean] (default false)

If true, create intermediate directories to your data file.

callback Function

callback that takes error (if any)

Examples

io.convertDbfToData('path/to/data.dbf', 'path/to/data.csv', function(err){
  console.log(err)
})

io.convertDbfToData('path/to/data.dbf', 'path/to/create/to/data.csv', {makeDirectories: true}, function(err){
  console.log(err)
})

helpers

Various utility functions.

deepExtend(destination, source, [source2])

A more semantic convenience function. Delegates to helpers#extend and passes true as the first argument. Recursively merge the contents of two or more objects together into the first object.

parameter type description
destination Object

The object to modify

source Object

The object whose contents to take

source2 [Object]

You can add any number of objects as arguments.

Returns

Object :

result The merged object. Note that the destination object will always be modified.

Examples

var object1 = {
  apple: 0,
  banana: { weight: 52, price: 100 },
  cherry: 97
}
var object2 = {
  banana: { price: 200 },
  almond: 100
}
io.deepExtend(object1, object2)
console.log(object1)
//  {
//   apple: 0,
//   banana: {
//     weight: 52,
//     price: 200
//   },
//   cherry: 97,
//   almond: 100
// }

discernFileFormatter(fileName)

Returns a formatter that will format json data to file type specified by the extension in fileName. Used internally by .writeData and .writeDataSync.

parameter type description
fileName String

the name of the file

Returns

Object :

a formatter that can write the file

Examples

var formatter = io.discernFileFormatter('path/to/data.tsv');
var csv = formatter(json);

discernFormat(fileName)

Given a fileName return its file extension. Used internally by .discernPaser and .discernFileFormatter.

parameter type description
fileName String

the name of the file

Returns

String :

the file's extension

Examples

var format = io.discernFormat('path/to/data.csv')
console.log(format) // 'csv'

discernParser(fileName, options)

Given a fileName, optionally a delimiter, return a parser that can read that file as json. Parses as text if format not supported. Used internally by .readData and .readDataSync.

parameter type description
fileName String

the name of the file

options Object

optionally can take a delimiter value

Returns

Object :

a parser that can read the file

Examples

var parser = io.discernParser('path/to/data.csv');
var json = parser('path/to/data.csv');

var parser = io.discernParser('path/to/data.usv', {delimiter: '_'});
var json = parser('path/to/data.usv');

exists(fileName, callback)

Asynchronously test whether a file exists or not by using fs.access modified from https://github.com/nodejs/io.js/issues/1592#issuecomment-98392899.

parameter type description
fileName String

the name of the file

callback Function

has the signature (err, exists)

Examples

var exists = io.exists('path/to/data.tsv', function (err, exists) {
  console.log(exists) // `true` if the file exists, `false` if not.
})

existsSync(fileName)

Syncronous version of helpers#exists

parameter type description
fileName String

the name of the file

Returns

Boolean :

whether the file exists or not

Examples

var exists = io.existsSync('path/to/data.tsv')
console.log(exists) // `true` if file exists, `false` if not.

extend([deepExtend], destination, source, [source2])

A port of jQuery's extend. Merge the contents of two or more objects together into the first object. Supports deep extending with true as the first argument.

parameter type description
deepExtend [Boolean]

Set to true to merge recursively.

destination Object

The object to modify

source Object

The object whose contents to take

source2 [Object]

You can add any number of objects as arguments.

Returns

Object :

result The merged object. Note that the destination object will always be modified.

Examples

var mergedObj = io.extend({}, {name: 'indian-ocean'}, {alias: 'io'})
console.log(mergedObj)
// {
//   name: 'indian-ocean',
//   alias: 'io'
// }

var name = {name: 'indian-ocean'}
io.extend(name, {alias: 'io'})
console.log(name)
// {
//   name: 'indian-ocean',
//   alias: 'io'
// }
var object1 = {
  apple: 0,
  banana: { weight: 52, price: 100 },
  cherry: 97
}
var object2 = {
  banana: { price: 200 },
  almond: 100
}
io.extend(true, object1, object2)
console.log(object1)
//  {
//   apple: 0,
//   banana: {
//     weight: 52,
//     price: 200
//   },
//   cherry: 97,
//   almond: 100
// }

extensionMatches(fileName, extension)

Test whether a file name has the given extension

parameter type description
fileName String

The name of the file

extension String

The extension to test

Returns

Boolean :

whether The extension matched or not

Examples

var matches = io.extensionMatches('path/to/data.tsv', 'tsv')
console.log(matches) // `true`

makeDirectories(outPath, callback)

Asynchronously Create directories in a given file path

parameter type description
outPath String

The path to a file

callback Function

The function to do once this is done. Has signature of (err)

Examples

io.makeDirectories('path/to/create/to/data.tsv', function (err) {
  console.log(err) // null
})

makeDirectoriesSync(outPath)

Synchronous version of makeDirectories

parameter type description
outPath String

The path to a file

Examples

io.makeDirectories('path/to/create/to/data.tsv')

matches(fileName)

Test whether a file name matches a given matcher. Delegates to helpers#extensionMatches if matcher is a string, [helpers#matchRegExp`](helpers-matchregexp) if Regular Expression

parameter type description
fileName String

The name of the file

Returns

String :

matcher The string to match with

Examples

var matches = io.matches('path/to/data.tsv', 'tsv')
console.log(matches) // `true`

var matches = io.matches('.gitignore', /\.gitignore/)
console.log(matches) // `true`

matchRegExp(fileName, RegEx)

Test whether a string matches a given Regular Expression

parameter type description
fileName String

The name of the file

RegEx RegExp

The RegEx to match with

Returns

Boolean :

whether The string matches the RegEx

Examples

var matches = io.matchRegExp('.gitignore', /\.gitignore/)
console.log(matches) // `true`

readers

Functions to read data files.

readData(fileName, [options], callback)

Asynchronously read data given a path ending in the file format.

Supported formats / extensions:

  • .json Array of objects
  • .csv Comma-separated
  • .tsv Tab-separated
  • .psv Pipe-separated
  • .yaml or .yml Yaml file
  • .aml ArchieML
  • .txt Text file (a string)
  • other All others are read as a text file

Note: Does not currently support .dbf files. .yaml and .yml formats are read with js-yaml's .load method, which has no security checking. See js-yaml library for more secure options.

parameter type description
fileName String

the name of the file

options [Object]

optional See options below

options.parser [String or Function or Object]

optional This can be a string that is the file's delimiter or a function that returns the json. See parsers in library source for examples. For convenience, this can also take a dsv object such as dsv.dsv('_') or any object that has a parse method. Pre-version 1.1 this was called parseWith and to maintain support, that will still work.

callback Function

callback used when read data is read, takes error (if any) and the data read

Examples

io.readData('path/to/data.tsv', function (err, data) {
  console.log(data) // Json data
})

io.readData('path/to/data.usv', {parser: '_'}, function (err, data) {
  console.log(data) // Json data
})

var myParser = dsv.dsv('_')
io.readData('path/to/data.usv', {parser: myParser}, function (err, data) {
  console.log(data) // Json data
})

var naiveJsonLines = function (dataAsString) {
  return dataAsString.split('\n').map(function (row) { return JSON.parse(row) })
}
io.readData('path/to/data.jsonlines', {parser: naiveJsonLines}, function (err, data) {
  console.log(data) // Json data
})

readDataSync(fileName, [options])

Syncronous version of readers#readData

parameter type description
fileName String

the name of the file

options [Object]

optional See options below

options.parser [String or Function or Object]

optional This can be a string that is the file's delimiter or a function that returns the json. See parsers in library source for examples. For convenience, this can also take a dsv object such as dsv.dsv('_') or any object that has a parse method. Pre-version 1.1 this was called parseWith and to maintain support, that will still work.

Returns

Object :

the contents of the file as JSON

Examples

io.readDataSync('path/to/data.tsv')
console.log(data) // Json data

io.readDataSync('path/to/data.usv', {parser: '_'})
console.log(data) // Json data

var myParser = dsv.dsv('_')
io.readDataSync('path/to/data.usv', {parser: myParser})
console.log(data) // Json data

var naiveJsonLines = function(dataAsString)
  return dataAsString.split('\n').map(function (row) { return JSON.parse(row) })
}
io.readDataSync('path/to/data.jsonlines', {parser: naiveJsonLines})
console.log(data) // Json data

readDbf(fileName, callback)

Asynchronously read a dbf file.

parameter type description
fileName String

the name of the file

callback Function

callback used when read data is read, takes error (if any) and the data read

Examples

io.readDbf('path/to/data.dbf', function(err, data){
  console.log(data) // Json data
})

readdirFilter(dirPath, options, callback)

Get a list of a directory's files and folders if certain critera are met.

parameter type description
dirPath String

The directory to read from

options Object

Filter options, see below

options.include String or RegExp or Array<String> or Array<RegExp>

If given a string, return files that have that string as their extension. If given a Regular Expression, return the file that matches the pattern. Can also take a list of both. List matching behavior is described in includeAll.

options.exclude String or RegExp or Array<String> or Array<RegExp>

If given a string, return files that do not have that string as their extension. If given a Regular Expression, return the file that matches the pattern. Can also take a list of both. List matching behavior is described in excludeAll.

options.includeMatchAll [Boolean] (default false)

If true, require all include conditions to be met for a file to be included.

options.excludeMatchAll [Boolean] (default false)

If true, require all exclude conditions to be met for a file to be excluded.

options.fullPath [Boolean] (default false)

If true the full path of the file, otherwise return just the file name.

options.skipFiles [Boolean] (default false)

If true, only include directories.

options.skipDirectories [Boolean] (default false)

If true, only include files.

callback Function

Callback fired with signature of (err, files) where files is a list of matching file names.

Examples

// dir contains `data-0.tsv`, `data-0.json`, `data-0.csv`, `data-1.csv`, `.hidden-file`
io.readdirFilter('path/to/files', {include: 'csv'}, function(err, files){
  console.log(files) // ['data-0.csv', 'data-1.csv']
})

io.readdirFilter('path/to/files', {include: [/^data/], exclude: ['csv', 'json']}, , function(err, files){
  console.log(files) // ['path/to/files/data-0.csv', 'path/to/files/data-1.csv', 'path/to/files/data-0.tsv']
})

readdirFilterSync(dirPath, options)

Synchronously get a list of a directory's files and folders if certain critera are met.

parameter type description
dirPath String

The directory to read from

options Object

filter options, see below

options.include String or RegExp or Array<String> or Array<RegExp>

if given a string, return files that have that string as their extension. If given a Regular Expression, return the file that matches the pattern. Can also take a list of both. List matching behavior is described in includeAll.

options.exclude String or RegExp or Array<String> or Array<RegExp>

if given a string, return files that do not have that string as their extension. If given a Regular Expression, return the file that matches the pattern. Can also take a list of both. List matching behavior is described in excludeAll.

options.includeMatchAll [Boolean] (default false)

If true, require all include conditions to be met for a file to be included.

options.excludeMatchAll [Boolean] (default false)

If true, require all exclude conditions to be met for a file to be excluded.

options.fullPath [Boolean] (default false)

If true the full path of the file, otherwise return just the file name.

options.skipFiles [Boolean] (default false)

If true, only include directories.

options.skipDirectories [Boolean] (default false)

If true, only include files.

Returns

Array<String> :

The matching file names

Examples

// dir contains `data-0.tsv`, `data-0.json`, `data-0.csv`, `data-1.csv`, `.hidden-file`
var files = io.readdirFilterSync('path/to/files', {include: 'csv'})
console.log(files) // ['data-0.csv', 'data-1.csv']

var files = io.readdirFilterSync('path/to/files', {include: [/^data/], exclude: 'json', fullPath: true})
console.log(files) // ['path/to/files/data-0.csv', 'path/to/files/data-1.csv', 'path/to/files/data-0.tsv']

var files = io.readdirFilterSync('path/to/files', {include: [/^data/, 'json'], fullPath: true, includeMatchAll: true})
console.log(files) // ['path/to/files/data-0.json', 'path/to/files/data-1.json']

shorthandReaders

Convenience functions to load in a file with a specific type. This is equivalent to using readData and specifying a parser.

readAml(fileName, callback)

Asynchronously read an ArchieMl file.

parameter type description
fileName String

the name of the file

callback Function

callback used when read data is read, takes error (if any) and the data read

Examples

io.readAml('path/to/data.aml', function(err, data){
  console.log(data) // json data
})

readAmlSync(fileName)

Synchronously read an ArchieML file.

parameter type description
fileName String

the name of the file

Returns

Object :

the contents of the file as a string

Examples

var data = io.readAmlSync('path/to/data.aml')
console.log(data) // json data

readCsv(fileName, callback)

Asynchronously read a comma-separated value file.

parameter type description
fileName String

the name of the file

callback Function

callback used when read data is read, takes error (if any) and the data read

Examples

io.readCsv('path/to/data.csv', function(err, data){
  console.log(data) // Json data
})

readCsvSync(fileName)

Synchronously read a comma-separated value file.

parameter type description
fileName String

the name of the file

Returns

Array :

the contents of the file as JSON

Examples

var data = io.readCsvSync('path/to/data.csv')
console.log(data) // Json data

readJson(fileName, callback)

Asynchronously read a JSON file.

parameter type description
fileName String

the name of the file

callback Function

callback used when read data is read, takes error (if any) and the data read

Examples

io.readJson('path/to/data.json', function(err, data){
  console.log(data) // Json data
})

readJsonSync(fileName)

Synchronously read a JSON file.

parameter type description
fileName String

the name of the file

Returns

Array :

the contents of the file as JSON

Examples

var data = io.readJsonSync('path/to/data.json')
console.log(data) // Json data

readPsv(fileName, callback)

Asynchronously read a pipe-separated value file.

parameter type description
fileName String

the name of the file

callback Function

callback used when read data is read, takes error (if any) and the data read

Examples

io.readPsv('path/to/data.psv', function(err, data){
  console.log(data) // Json data
})

readPsvSync(fileName)

Synchronously read a pipe-separated value file.

parameter type description
fileName String

the name of the file

Returns

Array :

the contents of the file as JSON

Examples

var data = io.readPsvSync('path/to/data.psv')
console.log(data) // Json data

readTsv(fileName, callback)

Asynchronously read a tab-separated value file.

parameter type description
fileName String

the name of the file

callback Function

callback used when read data is read, takes error (if any) and the data read

Examples

io.readTsv('path/to/data.tsv', function(err, data){
  console.log(data) // Json data
})

readTsvSync(fileName)

Synchronously read a tab-separated value file.

parameter type description
fileName String

the name of the file

Returns

Array :

the contents of the file as JSON

Examples

var data = io.readTsvSync('path/to/data.tsv')
console.log(data) // Json data

readTxt(fileName, callback)

Asynchronously read a text file.

parameter type description
fileName String

the name of the file

callback Function

callback used when read data is read, takes error (if any) and the data read

Examples

io.readTxt('path/to/data.txt', function(err, data){
  console.log(data) // string data
})

readTxtSync(fileName)

Synchronously read a text file.

parameter type description
fileName String

the name of the file

Returns

Array :

the contents of the file as a string

Examples

var data = io.readTxtSync('path/to/data.txt')
console.log(data) // string data

readYaml(fileName, callback)

Asynchronously read a yaml file. Note: uses js-yaml's .load method, which has no security checking for functions. Use the js-yaml library if you require stricter security.

parameter type description
fileName String

the name of the file

callback Function

callback used when read data is read, takes error (if any) and the data read

Examples

// Can be `.yaml` or `.yml` extension
io.readYaml('path/to/data.yaml', function(err, data){
  console.log(data) // json data
})

readYamlSync(fileName)

Synchronously read a yaml file. Note: uses js-yaml's .load method, which has no security checking for functions. Use the js-yaml library if you require stricter security.

parameter type description
fileName String

the name of the file

Returns

Array :

the contents of the file as a string

Examples

// Can be `.yaml` or `.yml` extension
var data = io.readYamlSync('path/to/data.yaml')
console.log(data) // json data

writers

Functions to write data files.

appendData(fileName, data, [options], callback)

Append to an existing data object, creating a new file if one does not exist. For tabular formats, data must be an array of flat objects (cannot contain nested objects or arrays).

Supported formats:

  • .json Array of objects
  • .csv Comma-separated
  • .tsv Tab-separated
  • .psv Pipe-separated
  • .yaml or .yml Yaml

Note: Does not currently support .dbf files.

parameter type description
fileName String

the name of the file

data Object

the data to write

options [Object]

Optional config object, see below

options.makeDirectories [Boolean] (default false)

If true, create intermediate directories to your data file.

callback Function

callback of (err, data) where err is any error and data is the data that was written out

Examples

io.writeAppendData('path/to/data.json', jsonData, function(err){
  console.log(err)
})

io.writeAppendData('path/to/create/to/data.csv', flatJsonData, {makeDirectories: true}, function(err){
  console.log(err)
})

appendDataSync(fileName, [options], data)

Synchronous version of writers#appendData. See that function for supported formats

parameter type description
fileName String

the name of the file

options [Object]

Optional config object, see below

options.makeDirectories [Boolean] (default false)

If true, create intermediate directories to your data file.

data Object

the data to write

Returns

Object :

the data that was written

Examples

io.writeAppendDataSync('path/to/data.json', jsonData)

io.writeAppendDataSync('path/to/create/to/data.csv', flatJsonData, {makeDirectories: true})

writeData(fileName, data, [options], callback)

Write the data object, inferring the file format from the file ending specified in fileName.

Supported formats:

  • .json Array of objects
  • .csv Comma-separated
  • .tsv Tab-separated
  • .psv Pipe-separated
  • .yaml Yaml file
  • .yml Yaml file
  • .dbf Database file, commonly used in ESRI-shapefile format.

Note: .yaml and .yml files are written with .dump, which has no security checking. See js-yaml for more secure optins.

parameter type description
fileName String

the name of the file

data Object

the data to write

options [Object]

Optional config object, see below

options.makeDirectories [Boolean] (default false)

If true, create intermediate directories to your data file.

callback Function

callback of (err, data) where err is any error and data is the data that was written out

Examples

io.writeData('path/to/data.json', jsonData, function(err){
  console.log(err)
})

io.writeData('path/to/create/to/data.csv', flatJsonData, {makeDirectories: true}, function(err){
  console.log(err)
})

writeDataSync(fileName, [options], data)

Syncronous version of writers#writeData

parameter type description
fileName String

the name of the file

options [Object]

Optional config object, see below

options.makeDirectories [Boolean] (default false)

If true, create intermediate directories to your data file.

data Object

the data to write

Returns

Object :

the data that was written

Examples

io.writeDataSync('path/to/data.json', jsonData)

io.writeDataSync('path/to/create/to/data.csv', flatJsonData, {makeDirectories: true})