From 5653ee6cda932651ceec7372e3214291b6448a5b Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Thu, 14 Dec 2017 19:14:02 -0200 Subject: [PATCH] Merge pull request #9134 from RocketChat/wrap-importers-in-try [FIX] Importers not recovering when an error occurs --- .../server/importer.js | 268 +++++++-------- .../server/importer.js | 313 +++++++++--------- .../server/importer.js | 275 +++++++-------- .../server/importer.js | 81 ++--- .../server/importer.js | 22 +- 5 files changed, 496 insertions(+), 463 deletions(-) diff --git a/packages/rocketchat-importer-csv/server/importer.js b/packages/rocketchat-importer-csv/server/importer.js index 11568c5d52a5..89710bad7567 100644 --- a/packages/rocketchat-importer-csv/server/importer.js +++ b/packages/rocketchat-importer-csv/server/importer.js @@ -169,169 +169,175 @@ export class CsvImporter extends Base { const startedByUserId = Meteor.userId(); Meteor.defer(() => { super.updateProgress(ProgressStep.IMPORTING_USERS); - //Import the users - for (const u of this.users.users) { - if (!u.do_import) { - continue; - } - - Meteor.runAsUser(startedByUserId, () => { - let existantUser = RocketChat.models.Users.findOneByEmailAddress(u.email); - //If we couldn't find one by their email address, try to find an existing user by their username - if (!existantUser) { - existantUser = RocketChat.models.Users.findOneByUsername(u.username); - } - - if (existantUser) { - //since we have an existing user, let's try a few things - u.rocketId = existantUser._id; - RocketChat.models.Users.update({ _id: u.rocketId }, { $addToSet: { importIds: u.id } }); - } else { - const userId = Accounts.createUser({ email: u.email, password: Date.now() + u.name + u.email.toUpperCase() }); - Meteor.runAsUser(userId, () => { - Meteor.call('setUsername', u.username, {joinDefaultChannelsSilenced: true}); - RocketChat.models.Users.setName(userId, u.name); - RocketChat.models.Users.update({ _id: userId }, { $addToSet: { importIds: u.id } }); - u.rocketId = userId; - }); + try { + //Import the users + for (const u of this.users.users) { + if (!u.do_import) { + continue; } - super.addCountCompleted(1); - }); - } - this.collection.update({ _id: this.users._id }, { $set: { 'users': this.users.users }}); + Meteor.runAsUser(startedByUserId, () => { + let existantUser = RocketChat.models.Users.findOneByEmailAddress(u.email); - //Import the channels - super.updateProgress(ProgressStep.IMPORTING_CHANNELS); - for (const c of this.channels.channels) { - if (!c.do_import) { - continue; - } + //If we couldn't find one by their email address, try to find an existing user by their username + if (!existantUser) { + existantUser = RocketChat.models.Users.findOneByUsername(u.username); + } - Meteor.runAsUser(startedByUserId, () => { - const existantRoom = RocketChat.models.Rooms.findOneByName(c.name); - //If the room exists or the name of it is 'general', then we don't need to create it again - if (existantRoom || c.name.toUpperCase() === 'GENERAL') { - c.rocketId = c.name.toUpperCase() === 'GENERAL' ? 'GENERAL' : existantRoom._id; - RocketChat.models.Rooms.update({ _id: c.rocketId }, { $addToSet: { importIds: c.id } }); - } else { - //Find the rocketchatId of the user who created this channel - let creatorId = startedByUserId; - for (const u of this.users.users) { - if (u.username === c.creator && u.do_import) { - creatorId = u.rocketId; - } + if (existantUser) { + //since we have an existing user, let's try a few things + u.rocketId = existantUser._id; + RocketChat.models.Users.update({ _id: u.rocketId }, { $addToSet: { importIds: u.id } }); + } else { + const userId = Accounts.createUser({ email: u.email, password: Date.now() + u.name + u.email.toUpperCase() }); + Meteor.runAsUser(userId, () => { + Meteor.call('setUsername', u.username, {joinDefaultChannelsSilenced: true}); + RocketChat.models.Users.setName(userId, u.name); + RocketChat.models.Users.update({ _id: userId }, { $addToSet: { importIds: u.id } }); + u.rocketId = userId; + }); } - //Create the channel - Meteor.runAsUser(creatorId, () => { - const roomInfo = Meteor.call(c.isPrivate ? 'createPrivateGroup' : 'createChannel', c.name, c.members); - c.rocketId = roomInfo.rid; - }); + super.addCountCompleted(1); + }); + } + this.collection.update({ _id: this.users._id }, { $set: { 'users': this.users.users }}); - RocketChat.models.Rooms.update({ _id: c.rocketId }, { $addToSet: { importIds: c.id } }); + //Import the channels + super.updateProgress(ProgressStep.IMPORTING_CHANNELS); + for (const c of this.channels.channels) { + if (!c.do_import) { + continue; } - super.addCountCompleted(1); - }); - } - this.collection.update({ _id: this.channels._id }, { $set: { 'channels': this.channels.channels }}); - - //If no channels file, collect channel map from DB for message-only import - if (this.channels.channels.length === 0) { - for (const cname of this.messages.keys()) { Meteor.runAsUser(startedByUserId, () => { - const existantRoom = RocketChat.models.Rooms.findOneByName(cname); - if (existantRoom || cname.toUpperCase() === 'GENERAL') { - this.channels.channels.push({ - id: cname.replace('.', '_'), - name: cname, - rocketId: (cname.toUpperCase() === 'GENERAL' ? 'GENERAL' : existantRoom._id), - do_import: true + const existantRoom = RocketChat.models.Rooms.findOneByName(c.name); + //If the room exists or the name of it is 'general', then we don't need to create it again + if (existantRoom || c.name.toUpperCase() === 'GENERAL') { + c.rocketId = c.name.toUpperCase() === 'GENERAL' ? 'GENERAL' : existantRoom._id; + RocketChat.models.Rooms.update({ _id: c.rocketId }, { $addToSet: { importIds: c.id } }); + } else { + //Find the rocketchatId of the user who created this channel + let creatorId = startedByUserId; + for (const u of this.users.users) { + if (u.username === c.creator && u.do_import) { + creatorId = u.rocketId; + } + } + + //Create the channel + Meteor.runAsUser(creatorId, () => { + const roomInfo = Meteor.call(c.isPrivate ? 'createPrivateGroup' : 'createChannel', c.name, c.members); + c.rocketId = roomInfo.rid; }); + + RocketChat.models.Rooms.update({ _id: c.rocketId }, { $addToSet: { importIds: c.id } }); } + + super.addCountCompleted(1); }); } - } + this.collection.update({ _id: this.channels._id }, { $set: { 'channels': this.channels.channels }}); + + //If no channels file, collect channel map from DB for message-only import + if (this.channels.channels.length === 0) { + for (const cname of this.messages.keys()) { + Meteor.runAsUser(startedByUserId, () => { + const existantRoom = RocketChat.models.Rooms.findOneByName(cname); + if (existantRoom || cname.toUpperCase() === 'GENERAL') { + this.channels.channels.push({ + id: cname.replace('.', '_'), + name: cname, + rocketId: (cname.toUpperCase() === 'GENERAL' ? 'GENERAL' : existantRoom._id), + do_import: true + }); + } + }); + } + } + + //If no users file, collect user map from DB for message-only import + if (this.users.users.length === 0) { + for (const [ch, messagesMap] of this.messages.entries()) { + const csvChannel = this.getChannelFromName(ch); + if (!csvChannel || !csvChannel.do_import) { + continue; + } + Meteor.runAsUser(startedByUserId, () => { + for (const msgs of messagesMap.values()) { + for (const msg of msgs.messages) { + if (!this.getUserFromUsername(msg.username)) { + const user = RocketChat.models.Users.findOneByUsername(msg.username); + if (user) { + this.users.users.push({ + rocketId: user._id, + username: user.username + }); + } + } + } + } + }); + } + } - //If no users file, collect user map from DB for message-only import - if (this.users.users.length === 0) { + //Import the Messages + super.updateProgress(ProgressStep.IMPORTING_MESSAGES); for (const [ch, messagesMap] of this.messages.entries()) { const csvChannel = this.getChannelFromName(ch); if (!csvChannel || !csvChannel.do_import) { continue; } + + const room = RocketChat.models.Rooms.findOneById(csvChannel.rocketId, { fields: { usernames: 1, t: 1, name: 1 } }); Meteor.runAsUser(startedByUserId, () => { - for (const msgs of messagesMap.values()) { + const timestamps = {}; + for (const [msgGroupData, msgs] of messagesMap.entries()) { + super.updateRecord({ 'messagesstatus': `${ ch }/${ msgGroupData }.${ msgs.messages.length }` }); for (const msg of msgs.messages) { - if (!this.getUserFromUsername(msg.username)) { - const user = RocketChat.models.Users.findOneByUsername(msg.username); - if (user) { - this.users.users.push({ - rocketId: user._id, - username: user.username - }); + if (isNaN(new Date(parseInt(msg.ts)))) { + this.logger.warn(`Timestamp on a message in ${ ch }/${ msgGroupData } is invalid`); + super.addCountCompleted(1); + continue; + } + + const creator = this.getUserFromUsername(msg.username); + if (creator) { + let suffix = ''; + if (timestamps[msg.ts] === undefined) { + timestamps[msg.ts] = 1; + } else { + suffix = `-${ timestamps[msg.ts] }`; + timestamps[msg.ts] += 1; } + const msgObj = { + _id: `csv-${ csvChannel.id }-${ msg.ts }${ suffix }`, + ts: new Date(parseInt(msg.ts)), + msg: msg.text, + rid: room._id, + u: { + _id: creator._id, + username: creator.username + } + }; + + RocketChat.sendMessage(creator, msgObj, room, true); } + + super.addCountCompleted(1); } } }); } - } - - //Import the Messages - super.updateProgress(ProgressStep.IMPORTING_MESSAGES); - for (const [ch, messagesMap] of this.messages.entries()) { - const csvChannel = this.getChannelFromName(ch); - if (!csvChannel || !csvChannel.do_import) { - continue; - } - - const room = RocketChat.models.Rooms.findOneById(csvChannel.rocketId, { fields: { usernames: 1, t: 1, name: 1 } }); - Meteor.runAsUser(startedByUserId, () => { - const timestamps = {}; - for (const [msgGroupData, msgs] of messagesMap.entries()) { - super.updateRecord({ 'messagesstatus': `${ ch }/${ msgGroupData }.${ msgs.messages.length }` }); - for (const msg of msgs.messages) { - if (isNaN(new Date(parseInt(msg.ts)))) { - this.logger.warn(`Timestamp on a message in ${ ch }/${ msgGroupData } is invalid`); - super.addCountCompleted(1); - continue; - } - - const creator = this.getUserFromUsername(msg.username); - if (creator) { - let suffix = ''; - if (timestamps[msg.ts] === undefined) { - timestamps[msg.ts] = 1; - } else { - suffix = `-${ timestamps[msg.ts] }`; - timestamps[msg.ts] += 1; - } - const msgObj = { - _id: `csv-${ csvChannel.id }-${ msg.ts }${ suffix }`, - ts: new Date(parseInt(msg.ts)), - msg: msg.text, - rid: room._id, - u: { - _id: creator._id, - username: creator.username - } - }; - - RocketChat.sendMessage(creator, msgObj, room, true); - } - - super.addCountCompleted(1); - } - } - }); + super.updateProgress(ProgressStep.FINISHING); + super.updateProgress(ProgressStep.DONE); + } catch (e) { + this.logger.error(e); + super.updateProgress(ProgressStep.ERROR); } - super.updateProgress(ProgressStep.FINISHING); - super.updateProgress(ProgressStep.DONE); const timeTook = Date.now() - started; this.logger.log(`CSV Import took ${ timeTook } milliseconds.`); }); diff --git a/packages/rocketchat-importer-hipchat-enterprise/server/importer.js b/packages/rocketchat-importer-hipchat-enterprise/server/importer.js index 7e566662a3d6..bc74cd5ff5ba 100644 --- a/packages/rocketchat-importer-hipchat-enterprise/server/importer.js +++ b/packages/rocketchat-importer-hipchat-enterprise/server/importer.js @@ -240,193 +240,200 @@ export class HipChatEnterpriseImporter extends Base { const startedByUserId = Meteor.userId(); Meteor.defer(() => { super.updateProgress(ProgressStep.IMPORTING_USERS); - //Import the users - for (const u of this.users.users) { - this.logger.debug(`Starting the user import: ${ u.username } and are we importing them? ${ u.do_import }`); - if (!u.do_import) { - continue; - } - Meteor.runAsUser(startedByUserId, () => { - let existantUser = RocketChat.models.Users.findOneByEmailAddress(u.email); + try { + //Import the users + for (const u of this.users.users) { + this.logger.debug(`Starting the user import: ${ u.username } and are we importing them? ${ u.do_import }`); + if (!u.do_import) { + continue; + } + + Meteor.runAsUser(startedByUserId, () => { + let existantUser = RocketChat.models.Users.findOneByEmailAddress(u.email); + + //If we couldn't find one by their email address, try to find an existing user by their username + if (!existantUser) { + existantUser = RocketChat.models.Users.findOneByUsername(u.username); + } + + if (existantUser) { + //since we have an existing user, let's try a few things + u.rocketId = existantUser._id; + RocketChat.models.Users.update({ _id: u.rocketId }, { $addToSet: { importIds: u.id } }); + } else { + const userId = Accounts.createUser({ email: u.email, password: Date.now() + u.name + u.email.toUpperCase() }); + Meteor.runAsUser(userId, () => { + Meteor.call('setUsername', u.username, {joinDefaultChannelsSilenced: true}); + //TODO: Use moment timezone to calc the time offset - Meteor.call 'userSetUtcOffset', user.tz_offset / 3600 + RocketChat.models.Users.setName(userId, u.name); + //TODO: Think about using a custom field for the users "title" field + + if (u.avatar) { + Meteor.call('setAvatarFromService', `data:image/png;base64,${ u.avatar }`); + } + + //Deleted users are 'inactive' users in Rocket.Chat + if (u.deleted) { + Meteor.call('setUserActiveStatus', userId, false); + } + + RocketChat.models.Users.update({ _id: userId }, { $addToSet: { importIds: u.id } }); + u.rocketId = userId; + }); + } + + super.addCountCompleted(1); + }); + } + this.collection.update({ _id: this.users._id }, { $set: { 'users': this.users.users }}); - //If we couldn't find one by their email address, try to find an existing user by their username - if (!existantUser) { - existantUser = RocketChat.models.Users.findOneByUsername(u.username); + //Import the channels + super.updateProgress(ProgressStep.IMPORTING_CHANNELS); + for (const c of this.channels.channels) { + if (!c.do_import) { + continue; } - if (existantUser) { - //since we have an existing user, let's try a few things - u.rocketId = existantUser._id; - RocketChat.models.Users.update({ _id: u.rocketId }, { $addToSet: { importIds: u.id } }); - } else { - const userId = Accounts.createUser({ email: u.email, password: Date.now() + u.name + u.email.toUpperCase() }); - Meteor.runAsUser(userId, () => { - Meteor.call('setUsername', u.username, {joinDefaultChannelsSilenced: true}); - //TODO: Use moment timezone to calc the time offset - Meteor.call 'userSetUtcOffset', user.tz_offset / 3600 - RocketChat.models.Users.setName(userId, u.name); - //TODO: Think about using a custom field for the users "title" field - - if (u.avatar) { - Meteor.call('setAvatarFromService', `data:image/png;base64,${ u.avatar }`); + Meteor.runAsUser(startedByUserId, () => { + const existantRoom = RocketChat.models.Rooms.findOneByName(c.name); + //If the room exists or the name of it is 'general', then we don't need to create it again + if (existantRoom || c.name.toUpperCase() === 'GENERAL') { + c.rocketId = c.name.toUpperCase() === 'GENERAL' ? 'GENERAL' : existantRoom._id; + RocketChat.models.Rooms.update({ _id: c.rocketId }, { $addToSet: { importIds: c.id } }); + } else { + //Find the rocketchatId of the user who created this channel + let creatorId = startedByUserId; + for (const u of this.users.users) { + if (u.id === c.creator && u.do_import) { + creatorId = u.rocketId; + } } - //Deleted users are 'inactive' users in Rocket.Chat - if (u.deleted) { - Meteor.call('setUserActiveStatus', userId, false); - } + //Create the channel + Meteor.runAsUser(creatorId, () => { + const roomInfo = Meteor.call(c.isPrivate ? 'createPrivateGroup' : 'createChannel', c.name, []); + c.rocketId = roomInfo.rid; + }); - RocketChat.models.Users.update({ _id: userId }, { $addToSet: { importIds: u.id } }); - u.rocketId = userId; - }); + RocketChat.models.Rooms.update({ _id: c.rocketId }, { $set: { ts: c.created, topic: c.topic }, $addToSet: { importIds: c.id } }); + } + + super.addCountCompleted(1); + }); + } + this.collection.update({ _id: this.channels._id }, { $set: { 'channels': this.channels.channels }}); + + //Import the Messages + super.updateProgress(ProgressStep.IMPORTING_MESSAGES); + for (const [ch, messagesMap] of this.messages.entries()) { + const hipChannel = this.getChannelFromRoomIdentifier(ch); + if (!hipChannel.do_import) { + continue; } - super.addCountCompleted(1); - }); - } - this.collection.update({ _id: this.users._id }, { $set: { 'users': this.users.users }}); + const room = RocketChat.models.Rooms.findOneById(hipChannel.rocketId, { fields: { usernames: 1, t: 1, name: 1 } }); + Meteor.runAsUser(startedByUserId, () => { + for (const [msgGroupData, msgs] of messagesMap.entries()) { + super.updateRecord({ 'messagesstatus': `${ ch }/${ msgGroupData }.${ msgs.messages.length }` }); + for (const msg of msgs.messages) { + if (isNaN(msg.ts)) { + this.logger.warn(`Timestamp on a message in ${ ch }/${ msgGroupData } is invalid`); + super.addCountCompleted(1); + continue; + } - //Import the channels - super.updateProgress(ProgressStep.IMPORTING_CHANNELS); - for (const c of this.channels.channels) { - if (!c.do_import) { - continue; - } + const creator = this.getRocketUserFromUserId(msg.userId); + if (creator) { + switch (msg.type) { + case 'user': + RocketChat.sendMessage(creator, { + _id: msg.id, + ts: msg.ts, + msg: msg.text, + rid: room._id, + u: { + _id: creator._id, + username: creator.username + } + }, room, true); + break; + case 'topic': + RocketChat.models.Messages.createRoomSettingsChangedWithTypeRoomIdMessageAndUser('room_changed_topic', room._id, msg.text, creator, { _id: msg.id, ts: msg.ts }); + break; + } + } - Meteor.runAsUser(startedByUserId, () => { - const existantRoom = RocketChat.models.Rooms.findOneByName(c.name); - //If the room exists or the name of it is 'general', then we don't need to create it again - if (existantRoom || c.name.toUpperCase() === 'GENERAL') { - c.rocketId = c.name.toUpperCase() === 'GENERAL' ? 'GENERAL' : existantRoom._id; - RocketChat.models.Rooms.update({ _id: c.rocketId }, { $addToSet: { importIds: c.id } }); - } else { - //Find the rocketchatId of the user who created this channel - let creatorId = startedByUserId; - for (const u of this.users.users) { - if (u.id === c.creator && u.do_import) { - creatorId = u.rocketId; + super.addCountCompleted(1); } } + }); + } - //Create the channel - Meteor.runAsUser(creatorId, () => { - const roomInfo = Meteor.call(c.isPrivate ? 'createPrivateGroup' : 'createChannel', c.name, []); - c.rocketId = roomInfo.rid; - }); - - RocketChat.models.Rooms.update({ _id: c.rocketId }, { $set: { ts: c.created, topic: c.topic }, $addToSet: { importIds: c.id } }); + //Import the Direct Messages + for (const [directMsgRoom, directMessagesMap] of this.directMessages.entries()) { + const hipUser = this.getUserFromDirectMessageIdentifier(directMsgRoom); + if (!hipUser.do_import) { + continue; } - super.addCountCompleted(1); - }); - } - this.collection.update({ _id: this.channels._id }, { $set: { 'channels': this.channels.channels }}); - - //Import the Messages - super.updateProgress(ProgressStep.IMPORTING_MESSAGES); - for (const [ch, messagesMap] of this.messages.entries()) { - const hipChannel = this.getChannelFromRoomIdentifier(ch); - if (!hipChannel.do_import) { - continue; - } + //Verify this direct message user's room is valid (confusing but idk how else to explain it) + if (!this.getRocketUserFromUserId(hipUser.id)) { + continue; + } - const room = RocketChat.models.Rooms.findOneById(hipChannel.rocketId, { fields: { usernames: 1, t: 1, name: 1 } }); - Meteor.runAsUser(startedByUserId, () => { - for (const [msgGroupData, msgs] of messagesMap.entries()) { - super.updateRecord({ 'messagesstatus': `${ ch }/${ msgGroupData }.${ msgs.messages.length }` }); + for (const [msgGroupData, msgs] of directMessagesMap.entries()) { + super.updateRecord({ 'messagesstatus': `${ directMsgRoom }/${ msgGroupData }.${ msgs.messages.length }` }); for (const msg of msgs.messages) { if (isNaN(msg.ts)) { - this.logger.warn(`Timestamp on a message in ${ ch }/${ msgGroupData } is invalid`); + this.logger.warn(`Timestamp on a message in ${ directMsgRoom }/${ msgGroupData } is invalid`); super.addCountCompleted(1); continue; } - const creator = this.getRocketUserFromUserId(msg.userId); - if (creator) { - switch (msg.type) { - case 'user': - RocketChat.sendMessage(creator, { - _id: msg.id, - ts: msg.ts, - msg: msg.text, - rid: room._id, - u: { - _id: creator._id, - username: creator.username - } - }, room, true); - break; - case 'topic': - RocketChat.models.Messages.createRoomSettingsChangedWithTypeRoomIdMessageAndUser('room_changed_topic', room._id, msg.text, creator, { _id: msg.id, ts: msg.ts }); - break; - } + //make sure the message sender is a valid user inside rocket.chat + const sender = this.getRocketUserFromUserId(msg.senderId); + if (!sender) { + continue; } - super.addCountCompleted(1); - } - } - }); - } - - //Import the Direct Messages - for (const [directMsgRoom, directMessagesMap] of this.directMessages.entries()) { - const hipUser = this.getUserFromDirectMessageIdentifier(directMsgRoom); - if (!hipUser.do_import) { - continue; - } - - //Verify this direct message user's room is valid (confusing but idk how else to explain it) - if (!this.getRocketUserFromUserId(hipUser.id)) { - continue; - } - - for (const [msgGroupData, msgs] of directMessagesMap.entries()) { - super.updateRecord({ 'messagesstatus': `${ directMsgRoom }/${ msgGroupData }.${ msgs.messages.length }` }); - for (const msg of msgs.messages) { - if (isNaN(msg.ts)) { - this.logger.warn(`Timestamp on a message in ${ directMsgRoom }/${ msgGroupData } is invalid`); - super.addCountCompleted(1); - continue; - } - - //make sure the message sender is a valid user inside rocket.chat - const sender = this.getRocketUserFromUserId(msg.senderId); - if (!sender) { - continue; - } + //make sure the receiver of the message is a valid rocket.chat user + const receiver = this.getRocketUserFromUserId(msg.receiverId); + if (!receiver) { + continue; + } - //make sure the receiver of the message is a valid rocket.chat user - const receiver = this.getRocketUserFromUserId(msg.receiverId); - if (!receiver) { - continue; - } + let room = RocketChat.models.Rooms.findOneById([receiver._id, sender._id].sort().join('')); + if (!room) { + Meteor.runAsUser(sender._id, () => { + const roomInfo = Meteor.call('createDirectMessage', receiver.username); + room = RocketChat.models.Rooms.findOneById(roomInfo.rid); + }); + } - let room = RocketChat.models.Rooms.findOneById([receiver._id, sender._id].sort().join('')); - if (!room) { Meteor.runAsUser(sender._id, () => { - const roomInfo = Meteor.call('createDirectMessage', receiver.username); - room = RocketChat.models.Rooms.findOneById(roomInfo.rid); + RocketChat.sendMessage(sender, { + _id: msg.id, + ts: msg.ts, + msg: msg.text, + rid: room._id, + u: { + _id: sender._id, + username: sender.username + } + }, room, true); }); } - - Meteor.runAsUser(sender._id, () => { - RocketChat.sendMessage(sender, { - _id: msg.id, - ts: msg.ts, - msg: msg.text, - rid: room._id, - u: { - _id: sender._id, - username: sender.username - } - }, room, true); - }); } } + + super.updateProgress(ProgressStep.FINISHING); + super.updateProgress(ProgressStep.DONE); + } catch (e) { + this.logger.error(e); + super.updateProgress(ProgressStep.ERROR); } - super.updateProgress(ProgressStep.FINISHING); - super.updateProgress(ProgressStep.DONE); const timeTook = Date.now() - started; this.logger.log(`HipChat Enterprise Import took ${ timeTook } milliseconds.`); }); diff --git a/packages/rocketchat-importer-hipchat/server/importer.js b/packages/rocketchat-importer-hipchat/server/importer.js index 072d720d0f44..6f2fcb57f1b3 100644 --- a/packages/rocketchat-importer-hipchat/server/importer.js +++ b/packages/rocketchat-importer-hipchat/server/importer.js @@ -41,7 +41,7 @@ export class HipChatImporter extends Base { if (entry.entryName.indexOf(this.roomPrefix) > -1) { let roomName = entry.entryName.split(this.roomPrefix)[1]; if (roomName === 'list.json') { - this.updateProgress(ProgressStep.PREPARING_CHANNELS); + super.updateProgress(ProgressStep.PREPARING_CHANNELS); const tempRooms = JSON.parse(entry.getData().toString()).rooms; tempRooms.forEach(room => { room.name = s.slugify(room.name); @@ -62,7 +62,7 @@ export class HipChatImporter extends Base { } else if (entry.entryName.indexOf(this.usersPrefix) > -1) { const usersName = entry.entryName.split(this.usersPrefix)[1]; if (usersName === 'list.json') { - this.updateProgress(ProgressStep.PREPARING_USERS); + super.updateProgress(ProgressStep.PREPARING_USERS); return tempUsers = JSON.parse(entry.getData().toString()).users; } else { return this.logger.warn(`Unexpected file in the ${ this.name } import: ${ entry.entryName }`); @@ -91,7 +91,7 @@ export class HipChatImporter extends Base { 'count.channels': tempRooms.length }); this.addCountToTotal(tempRooms.length); - this.updateProgress(ProgressStep.PREPARING_MESSAGES); + super.updateProgress(ProgressStep.PREPARING_MESSAGES); let messagesCount = 0; Object.keys(tempMessages).forEach(channel => { const messagesObj = tempMessages[channel]; @@ -132,7 +132,7 @@ export class HipChatImporter extends Base { this.addCountToTotal(messagesCount); if (tempUsers.length === 0 || tempRooms.length === 0 || messagesCount === 0) { this.logger.warn(`The loaded users count ${ tempUsers.length }, the loaded channels ${ tempRooms.length }, and the loaded messages ${ messagesCount }`); - this.updateProgress(ProgressStep.ERROR); + super.updateProgress(ProgressStep.ERROR); return this.getProgress(); } const selectionUsers = tempUsers.map(function(user) { @@ -142,13 +142,14 @@ export class HipChatImporter extends Base { return new SelectionChannel(room.room_id, room.name, room.is_archived, true, false); }); const selectionMessages = this.importRecord.count.messages; - this.updateProgress(ProgressStep.USER_SELECTION); + super.updateProgress(ProgressStep.USER_SELECTION); return new Selection(this.name, selectionUsers, selectionChannels, selectionMessages); } startImport(importSelection) { super.startImport(importSelection); const start = Date.now(); + importSelection.users.forEach(user => { this.users.users.forEach(u => { if (u.user_id === user.user_id) { @@ -157,156 +158,168 @@ export class HipChatImporter extends Base { }); }); this.collection.update({_id: this.users._id}, { $set: { 'users': this.users.users } }); + importSelection.channels.forEach(channel => this.channels.channels.forEach(c => c.room_id === channel.channel_id && (c.do_import = channel.do_import)) ); this.collection.update({ _id: this.channels._id }, { $set: { 'channels': this.channels.channels }}); + const startedByUserId = Meteor.userId(); Meteor.defer(() => { - this.updateProgress(ProgressStep.IMPORTING_USERS); - this.users.users.forEach(user => { - if (!user.do_import) { - return; - } - Meteor.runAsUser(startedByUserId, () => { - const existantUser = RocketChat.models.Users.findOneByEmailAddress(user.email); - if (existantUser) { - user.rocketId = existantUser._id; - this.userTags.push({ - hipchat: `@${ user.mention_name }`, - rocket: `@${ existantUser.username }` - }); - } else { - const userId = Accounts.createUser({ - email: user.email, - password: Date.now() + user.name + user.email.toUpperCase() - }); - user.rocketId = userId; - this.userTags.push({ - hipchat: `@${ user.mention_name }`, - rocket: `@${ user.mention_name }` - }); - Meteor.runAsUser(userId, () => { - Meteor.call('setUsername', user.mention_name, { - joinDefaultChannelsSilenced: true + super.updateProgress(ProgressStep.IMPORTING_USERS); + + try { + this.users.users.forEach(user => { + if (!user.do_import) { + return; + } + + Meteor.runAsUser(startedByUserId, () => { + const existantUser = RocketChat.models.Users.findOneByEmailAddress(user.email); + if (existantUser) { + user.rocketId = existantUser._id; + this.userTags.push({ + hipchat: `@${ user.mention_name }`, + rocket: `@${ existantUser.username }` }); - Meteor.call('setAvatarFromService', user.photo_url, undefined, 'url'); - return Meteor.call('userSetUtcOffset', parseInt(moment().tz(user.timezone).format('Z').toString().split(':')[0])); - }); - if (user.name != null) { - RocketChat.models.Users.setName(userId, user.name); - } - if (user.is_deleted) { - Meteor.call('setUserActiveStatus', userId, false); + } else { + const userId = Accounts.createUser({ + email: user.email, + password: Date.now() + user.name + user.email.toUpperCase() + }); + user.rocketId = userId; + this.userTags.push({ + hipchat: `@${ user.mention_name }`, + rocket: `@${ user.mention_name }` + }); + Meteor.runAsUser(userId, () => { + Meteor.call('setUsername', user.mention_name, { + joinDefaultChannelsSilenced: true + }); + Meteor.call('setAvatarFromService', user.photo_url, undefined, 'url'); + return Meteor.call('userSetUtcOffset', parseInt(moment().tz(user.timezone).format('Z').toString().split(':')[0])); + }); + if (user.name != null) { + RocketChat.models.Users.setName(userId, user.name); + } + if (user.is_deleted) { + Meteor.call('setUserActiveStatus', userId, false); + } } - } - return this.addCountCompleted(1); + return this.addCountCompleted(1); + }); }); - }); - this.collection.update({ _id: this.users._id }, { $set: { 'users': this.users.users }}); - this.updateProgress(ProgressStep.IMPORTING_CHANNELS); - this.channels.channels.forEach(channel => { - if (!channel.do_import) { - return; - } - Meteor.runAsUser(startedByUserId, () => { - channel.name = channel.name.replace(/ /g, ''); - const existantRoom = RocketChat.models.Rooms.findOneByName(channel.name); - if (existantRoom) { - channel.rocketId = existantRoom._id; - } else { - let userId = ''; - this.users.users.forEach(user => { - if (user.user_id === channel.owner_user_id) { - userId = user.rocketId; + + this.collection.update({ _id: this.users._id }, { $set: { 'users': this.users.users }}); + + super.updateProgress(ProgressStep.IMPORTING_CHANNELS); + this.channels.channels.forEach(channel => { + if (!channel.do_import) { + return; + } + Meteor.runAsUser(startedByUserId, () => { + channel.name = channel.name.replace(/ /g, ''); + const existantRoom = RocketChat.models.Rooms.findOneByName(channel.name); + if (existantRoom) { + channel.rocketId = existantRoom._id; + } else { + let userId = ''; + this.users.users.forEach(user => { + if (user.user_id === channel.owner_user_id) { + userId = user.rocketId; + } + }); + if (userId === '') { + this.logger.warn(`Failed to find the channel creator for ${ channel.name }, setting it to the current running user.`); + userId = startedByUserId; } - }); - if (userId === '') { - this.logger.warn(`Failed to find the channel creator for ${ channel.name }, setting it to the current running user.`); - userId = startedByUserId; + Meteor.runAsUser(userId, () => { + const returned = Meteor.call('createChannel', channel.name, []); + return channel.rocketId = returned.rid; + }); + RocketChat.models.Rooms.update({ + _id: channel.rocketId + }, { + $set: { + 'ts': new Date(channel.created * 1000) + } + }); } - Meteor.runAsUser(userId, () => { - const returned = Meteor.call('createChannel', channel.name, []); - return channel.rocketId = returned.rid; - }); - RocketChat.models.Rooms.update({ - _id: channel.rocketId - }, { - $set: { - 'ts': new Date(channel.created * 1000) - } - }); - } - return this.addCountCompleted(1); + return this.addCountCompleted(1); + }); }); - }); - this.collection.update({ - _id: this.channels._id - }, { - $set: { - 'channels': this.channels.channels - } - }); - this.updateProgress(ProgressStep.IMPORTING_MESSAGES); - const nousers = {}; - Object.keys(this.messages).forEach(channel => { - const messagesObj = this.messages[channel]; - Meteor.runAsUser(startedByUserId, () => { - const hipchatChannel = this.getHipChatChannelFromName(channel); - if (hipchatChannel != null ? hipchatChannel.do_import : undefined) { - const room = RocketChat.models.Rooms.findOneById(hipchatChannel.rocketId, { - fields: { - usernames: 1, - t: 1, - name: 1 - } - }); + this.collection.update({ _id: this.channels._id }, { $set: { 'channels': this.channels.channels }}); + + super.updateProgress(ProgressStep.IMPORTING_MESSAGES); + const nousers = {}; - Object.keys(messagesObj).forEach(date => { - const msgs = messagesObj[date]; - this.updateRecord({ - 'messagesstatus': `${ channel }/${ date }.${ msgs.messages.length }` + Object.keys(this.messages).forEach(channel => { + const messagesObj = this.messages[channel]; + Meteor.runAsUser(startedByUserId, () => { + const hipchatChannel = this.getHipChatChannelFromName(channel); + if (hipchatChannel != null ? hipchatChannel.do_import : undefined) { + const room = RocketChat.models.Rooms.findOneById(hipchatChannel.rocketId, { + fields: { + usernames: 1, + t: 1, + name: 1 + } }); - msgs.messages.forEach(message => { - if (message.from != null) { - const user = this.getRocketUser(message.from.user_id); - if (user != null) { - const msgObj = { - msg: this.convertHipChatMessageToRocketChat(message.message), - ts: new Date(message.date), - u: { - _id: user._id, - username: user.username - } - }; - RocketChat.sendMessage(user, msgObj, room, true); - } else if (!nousers[message.from.user_id]) { - nousers[message.from.user_id] = message.from; + Object.keys(messagesObj).forEach(date => { + const msgs = messagesObj[date]; + this.updateRecord({ + 'messagesstatus': `${ channel }/${ date }.${ msgs.messages.length }` + }); + + msgs.messages.forEach(message => { + if (message.from != null) { + const user = this.getRocketUser(message.from.user_id); + if (user != null) { + const msgObj = { + msg: this.convertHipChatMessageToRocketChat(message.message), + ts: new Date(message.date), + u: { + _id: user._id, + username: user.username + } + }; + RocketChat.sendMessage(user, msgObj, room, true); + } else if (!nousers[message.from.user_id]) { + nousers[message.from.user_id] = message.from; + } + } else if (!_.isArray(message)) { + console.warn('Please report the following:', message); } - } else if (!_.isArray(message)) { - console.warn('Please report the following:', message); - } - this.addCountCompleted(1); + this.addCountCompleted(1); + }); }); + } + }); + }); + + this.logger.warn('The following did not have users:', nousers); + super.updateProgress(ProgressStep.FINISHING); + + this.channels.channels.forEach(channel => { + if (channel.do_import && channel.is_archived) { + Meteor.runAsUser(startedByUserId, () => { + return Meteor.call('archiveRoom', channel.rocketId); }); } }); - }); - this.logger.warn('The following did not have users:', nousers); - this.updateProgress(ProgressStep.FINISHING); - this.channels.channels.forEach(channel => { - if (channel.do_import && channel.is_archived) { - Meteor.runAsUser(startedByUserId, () => { - return Meteor.call('archiveRoom', channel.rocketId); - }); - } - }); - this.updateProgress(ProgressStep.DONE); + + super.updateProgress(ProgressStep.DONE); + } catch (e) { + this.logger.error(e); + super.updateProgress(ProgressStep.ERROR); + } + const timeTook = Date.now() - start; return this.logger.log(`Import took ${ timeTook } milliseconds.`); }); + return this.getProgress(); } diff --git a/packages/rocketchat-importer-slack-users/server/importer.js b/packages/rocketchat-importer-slack-users/server/importer.js index c482a33716f5..0be7fbec801d 100644 --- a/packages/rocketchat-importer-slack-users/server/importer.js +++ b/packages/rocketchat-importer-slack-users/server/importer.js @@ -82,50 +82,57 @@ export class SlackUsersImporter extends Base { Meteor.defer(() => { super.updateProgress(ProgressStep.IMPORTING_USERS); - for (const u of this.users.users) { - if (!u.do_import) { - continue; - } - - Meteor.runAsUser(startedByUserId, () => { - const existantUser = RocketChat.models.Users.findOneByEmailAddress(u.email) || RocketChat.models.Users.findOneByUsername(u.username); - - let userId = existantUser._id; - if (existantUser) { - //since we have an existing user, let's try a few things - u.rocketId = existantUser._id; - RocketChat.models.Users.update({ _id: u.rocketId }, { $addToSet: { importIds: u.id } }); - - RocketChat.models.Users.setEmail(existantUser._id, u.email); - RocketChat.models.Users.setEmailVerified(existantUser._id, u.email); - } else { - userId = Accounts.createUser({ username: u.username + Random.id(), password: Date.now() + u.name + u.email.toUpperCase() }); + try { + for (const u of this.users.users) { + if (!u.do_import) { + continue; + } - if (!userId) { - console.warn('An error happened while creating a user.'); - return; + Meteor.runAsUser(startedByUserId, () => { + const existantUser = RocketChat.models.Users.findOneByEmailAddress(u.email) || RocketChat.models.Users.findOneByUsername(u.username); + + let userId; + if (existantUser) { + //since we have an existing user, let's try a few things + userId = existantUser._id; + u.rocketId = existantUser._id; + RocketChat.models.Users.update({ _id: u.rocketId }, { $addToSet: { importIds: u.id } }); + + RocketChat.models.Users.setEmail(existantUser._id, u.email); + RocketChat.models.Users.setEmailVerified(existantUser._id, u.email); + } else { + userId = Accounts.createUser({ username: u.username + Random.id(), password: Date.now() + u.name + u.email.toUpperCase() }); + + if (!userId) { + console.warn('An error happened while creating a user.'); + return; + } + + Meteor.runAsUser(userId, () => { + Meteor.call('setUsername', u.username, {joinDefaultChannelsSilenced: true}); + RocketChat.models.Users.setName(userId, u.name); + RocketChat.models.Users.update({ _id: userId }, { $addToSet: { importIds: u.id } }); + RocketChat.models.Users.setEmail(userId, u.email); + RocketChat.models.Users.setEmailVerified(userId, u.email); + u.rocketId = userId; + }); } - Meteor.runAsUser(userId, () => { - Meteor.call('setUsername', u.username, {joinDefaultChannelsSilenced: true}); - RocketChat.models.Users.setName(userId, u.name); - RocketChat.models.Users.update({ _id: userId }, { $addToSet: { importIds: u.id } }); - RocketChat.models.Users.setEmail(userId, u.email); - RocketChat.models.Users.setEmailVerified(userId, u.email); - u.rocketId = userId; - }); - } + if (this.admins.includes(u.user_id)) { + Meteor.call('setAdminStatus', userId, true); + } - if (this.admins.includes(u.user_id)) { - Meteor.call('setAdminStatus', userId, true); - } + super.addCountCompleted(1); + }); + } - super.addCountCompleted(1); - }); + super.updateProgress(ProgressStep.FINISHING); + super.updateProgress(ProgressStep.DONE); + } catch (e) { + this.logger.error(e); + super.updateProgress(ProgressStep.ERROR); } - super.updateProgress(ProgressStep.FINISHING); - super.updateProgress(ProgressStep.DONE); const timeTook = Date.now() - started; this.logger.log(`Slack Users Import took ${ timeTook } milliseconds.`); }); diff --git a/packages/rocketchat-importer-slack/server/importer.js b/packages/rocketchat-importer-slack/server/importer.js index ced00651a758..d5ff5d5dc404 100644 --- a/packages/rocketchat-importer-slack/server/importer.js +++ b/packages/rocketchat-importer-slack/server/importer.js @@ -32,13 +32,13 @@ export class SlackImporter extends Base { } if (entry.entryName === 'channels.json') { - this.updateProgress(ProgressStep.PREPARING_CHANNELS); + super.updateProgress(ProgressStep.PREPARING_CHANNELS); tempChannels = JSON.parse(entry.getData().toString()).filter(channel => channel.creator != null); return; } if (entry.entryName === 'users.json') { - this.updateProgress(ProgressStep.PREPARING_USERS); + super.updateProgress(ProgressStep.PREPARING_USERS); tempUsers = JSON.parse(entry.getData().toString()); tempUsers.forEach(user => { @@ -78,7 +78,7 @@ export class SlackImporter extends Base { this.addCountToTotal(tempChannels.length); // Insert the messages records - this.updateProgress(ProgressStep.PREPARING_MESSAGES); + super.updateProgress(ProgressStep.PREPARING_MESSAGES); let messagesCount = 0; Object.keys(tempMessages).forEach(channel => { @@ -109,14 +109,14 @@ export class SlackImporter extends Base { if ([tempUsers.length, tempChannels.length, messagesCount].some(e => e === 0)) { this.logger.warn(`The loaded users count ${ tempUsers.length }, the loaded channels ${ tempChannels.length }, and the loaded messages ${ messagesCount }`); console.log(`The loaded users count ${ tempUsers.length }, the loaded channels ${ tempChannels.length }, and the loaded messages ${ messagesCount }`); - this.updateProgress(ProgressStep.ERROR); + super.updateProgress(ProgressStep.ERROR); return this.getProgress(); } const selectionUsers = tempUsers.map(user => new SelectionUser(user.id, user.name, user.profile.email, user.deleted, user.is_bot, !user.is_bot)); const selectionChannels = tempChannels.map(channel => new SelectionChannel(channel.id, channel.name, channel.is_archived, true, false)); const selectionMessages = this.importRecord.count.messages; - this.updateProgress(ProgressStep.USER_SELECTION); + super.updateProgress(ProgressStep.USER_SELECTION); return new Selection(this.name, selectionUsers, selectionChannels, selectionMessages); } @@ -150,7 +150,7 @@ export class SlackImporter extends Base { const startedByUserId = Meteor.userId(); Meteor.defer(() => { try { - this.updateProgress(ProgressStep.IMPORTING_USERS); + super.updateProgress(ProgressStep.IMPORTING_USERS); this.users.users.forEach(user => { if (!user.do_import) { return; @@ -209,7 +209,7 @@ export class SlackImporter extends Base { }); this.collection.update({ _id: this.users._id }, { $set: { 'users': this.users.users }}); - this.updateProgress(ProgressStep.IMPORTING_CHANNELS); + super.updateProgress(ProgressStep.IMPORTING_CHANNELS); this.channels.channels.forEach(channel => { if (!channel.do_import) { return; @@ -265,7 +265,7 @@ export class SlackImporter extends Base { const missedTypes = {}; const ignoreTypes = { 'bot_add': true, 'file_comment': true, 'file_mention': true }; - this.updateProgress(ProgressStep.IMPORTING_MESSAGES); + super.updateProgress(ProgressStep.IMPORTING_MESSAGES); Object.keys(this.messages).forEach(channel => { const messagesObj = this.messages[channel]; @@ -431,7 +431,7 @@ export class SlackImporter extends Base { console.log('Missed import types:', missedTypes); } - this.updateProgress(ProgressStep.FINISHING); + super.updateProgress(ProgressStep.FINISHING); this.channels.channels.forEach(channel => { if (channel.do_import && channel.is_archived) { @@ -440,12 +440,12 @@ export class SlackImporter extends Base { }); } }); - this.updateProgress(ProgressStep.DONE); + super.updateProgress(ProgressStep.DONE); this.logger.log(`Import took ${ Date.now() - start } milliseconds.`); } catch (e) { this.logger.error(e); - this.updateProgress(ProgressStep.ERROR); + super.updateProgress(ProgressStep.ERROR); } });