关于node.js：使用brianc / node-postgres批量插入Postgres

Bulk insert into Postgres with brianc/node-postgres

我在使用pg的nodejs中有以下代码(https://github.com/brianc/node-postgres)
我为员工创建订阅的代码就是这样。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

client.query(
'INSERT INTO subscriptions (subscription_guid, employer_guid, employee_guid)
values ($1,$2,$3)', [
datasetArr[0].subscription_guid,
datasetArr[0].employer_guid,
datasetArr[0].employee_guid
],

FUNCTION(err, RESULT) {
done();

IF (err) {
set_response(500, err, res);
logger.error('error running query', err);
RETURN console.error('error running query', err);
}

logger.info('subscription with created');
set_response(201);

});

如您所知，datasetArr是一个数组。我想一次为多个员工创建大量订阅。但是我不想遍历整个数组。有没有办法用pg开箱即用？

相关讨论

我搜索了相同的问题，但尚未找到解决方案。
使用异步库，多次使用查询并进行必要的错误处理非常简单。

可能是此代码变体有所帮助。
(将10.000个小的json对象插入到空数据库中需要6秒钟)。

克里斯多夫

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

FUNCTION insertData(item,callback) {
client.query('INSERT INTO subscriptions (subscription_guid, employer_guid, employee_guid)
values ($1,$2,$3)', [
item.subscription_guid,
item.employer_guid,
item.employee_guid
],
FUNCTION(err,RESULT) {
// RETURN any err TO async.each iterator
callback(err);
})
}
async.each(datasetArr,insertData,FUNCTION(err) {
// Release the client TO the pg module
done();
IF (err) {
set_response(500, err, res);
logger.error('error running query', err);
RETURN console.error('error running query', err);
}
logger.info('subscription with created');
set_response(201);
})

要从NodeJS批量插入Postgresql，更好的选择是使用Postgres和pg-copy-streams提供的" COPY"命令。

来自以下代码段：https://gist.github.com/sairamkrish/477d20980611202f46a2d44648f7b14b

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

/*
Pseudo code - to serve as a help guide.
*/
const copyFrom = require('pg-copy-streams').from;
const Readable = require('stream').Readable;
const { Pool,Client } = require('pg');
const fs = require('fs');
const path = require('path');
const datasourcesConfigFilePath = path.join(__dirname,'..','..','server','datasources.json');
const datasources = JSON.parse(fs.readFileSync(datasourcesConfigFilePath, 'utf8'));

const pool = NEW Pool({
USER: datasources.PG.user,
host: datasources.PG.host,
DATABASE: datasources.PG.database,
password: datasources.PG.password,
port: datasources.PG.port,
});

export const bulkInsert = (employees) => {
pool.connect().then(client=>{
let done = () => {
client.release();
}
var stream = client.query(copyFrom('COPY employee (name,age,salary) FROM STDIN'));
var rs = NEW Readable;
let currentIndex = 0;
rs._read = FUNCTION () {
IF (currentIndex === employees.length) {
rs.push(NULL);
} ELSE {
let employee = employees[currentIndex];
rs.push(employee.name + '\\t' + employee.age + '\\t' + employee.salary + '\
');
currentIndex = currentIndex+1;
}
};
let onError = strErr => {
console.error('Something went wrong:', strErr);
done();
};
rs.on('error', onError);
stream.on('error', onError);
stream.on('end',done);
rs.pipe(stream);
});
}

此链接中说明的更详细的信息

在我看来，最好的方法是使用PostgreSQL json函数：

1
2
3
4
5
6
7
8
9

client.query('INSERT INTO table (columns) ' +
'SELECT m.* FROM json_populate_recordset(null::your_custom_type, $1) AS m',
[JSON.stringify(your_json_object_array)], FUNCTION(err, RESULT) {
IF(err) {
console.log(err);
} ELSE {
console.log(RESULT);
}
});