Implement CSV-based batch migration with per-project repository mapping

src/githubHelper.ts

@@ -103,16 +103,17 @@ export class GithubHelper {
 
     this.members = new Set<string>();
 
-    // TODO: won't work if ownerIsOrg is false
-    githubApi.orgs.listMembers(  {
-      org: this.githubOwner,
-    }).then(members => {
-      for (let member of members.data) {
-        this.members.add(member.login);
-      }
-    }).catch(err => {
-      console.error(`Failed to fetch organization members: ${err}`);
-    });
+    if (this.githubOwnerIsOrg) {
+      githubApi.orgs.listMembers(  {
+        org: this.githubOwner,
+      }).then(members => {
+        for (let member of members.data) {
+          this.members.add(member.login);
+        }
+      }).catch(err => {
+        console.error(`Failed to fetch organization members: ${err}`);
+      });
+    }
   }
 
   /*

src/index.ts

@@ -6,6 +6,7 @@ import {
 } from './githubHelper';
 import { GitlabHelper, GitLabIssue, GitLabMilestone } from './gitlabHelper';
 import settings from '../settings';
+import { readProjectsFromCsv } from './utils';
 
 import { Octokit as GitHubApi } from '@octokit/rest';
 import { throttling } from '@octokit/plugin-throttling';
@@ -94,17 +95,69 @@ const githubHelper = new GithubHelper(
   settings.useIssuesForAllMergeRequests
 );
 
-// If no project id is given in settings.js, just return
-// all of the projects that this user is associated with.
-if (!settings.gitlab.projectId) {
-  gitlabHelper.listProjects();
-} else {
-  // user has chosen a project
-  if (settings.github.recreateRepo === true) {
-    recreate();
+let projectMap: Map<number, [string, string]> = new Map();
+
+if (settings.csvImport?.projectMapCsv) {
+  console.log(`Loading projects from CSV: ${settings.csvImport.projectMapCsv}`);
+  projectMap = readProjectsFromCsv(
+    settings.csvImport.projectMapCsv,
+    settings.csvImport.gitlabProjectIdColumn,
+    settings.csvImport.gitlabProjectPathColumn,
+    settings.csvImport.githubProjectPathColumn
+  ); 
+  } else {
+    projectMap.set(settings.gitlab.projectId, ['', '']);
   }
-  migrate();
-}
+
+(async () => {
+  if (projectMap.size === 0 || (projectMap.size === 1 && projectMap.has(0))) {
+    await gitlabHelper.listProjects();
+  } else {
+    for (const projectId of projectMap.keys()) {
+      const paths = projectMap.get(projectId);
+
+      if (!paths) {
+        console.warn(`Warning: No paths found for project ID ${projectId}, skipping`);
+        continue;
+      }
+
+      const [gitlabPath, githubPath] = paths;
+
+      console.log(`\n\n${'='.repeat(60)}`);
+      if (gitlabPath) {
+        console.log(`Processing Project ID: ${projectId} ${gitlabPath} → GitHub: ${githubPath}`);
+      } else {
+        console.log(`Processing Project ID: ${projectId}`);
+      }
+      console.log(`${'='.repeat(60)}\n`);
+
+      settings.gitlab.projectId = projectId;
+      gitlabHelper.gitlabProjectId = projectId;
+
+      if (githubPath) {
+        const githubParts = githubPath.split('/');
+        if (githubParts.length === 2) {
+          settings.github.owner = githubParts[0];
+          settings.github.repo = githubParts[1];
+
+          githubHelper.githubOwner = githubParts[0];
+          githubHelper.githubRepo = githubParts[1];
+        } else {
+          settings.github.repo = githubPath;
+          githubHelper.githubRepo = githubPath;
+        }        
+      }
+
+      if (settings.github.recreateRepo === true) {
+        await recreate();
+      }
+      await migrate();
+    }
+  }
+})().catch(err => {
+  console.error('Migration failed:', err);
+  process.exit(1);
+});
 
 // ----------------------------------------------------------------------------
 

src/settings.ts

@@ -9,6 +9,12 @@ export default interface Settings {
   projectmap: {
     [key: string]: string;
   };
+  csvImport?:{
+    projectMapCsv?: string;
+    gitlabProjectIdColumn?: number;
+    gitlabProjectPathColumn?: number;
+    githubProjectPathColumn?: number;
+    }
   conversion: {
     useLowerCaseLabels: boolean;
     addIssueInformation: boolean;

src/utils.ts

@@ -3,13 +3,88 @@ import settings from '../settings';
 import * as mime from 'mime-types';
 import * as path from 'path';
 import * as crypto from 'crypto';
+import * as fs from 'fs';
 import S3 from 'aws-sdk/clients/s3';
 import { GitlabHelper } from './gitlabHelper';
 
 export const sleep = (milliseconds: number) => {
   return new Promise(resolve => setTimeout(resolve, milliseconds));
 };
 
+export const readProjectsFromCsv = (
+  filePath: string,
+  idColumn: number = 0,
+  gitlabPathColumn: number = 1,
+  githubPathColumn: number = 2
+): Map<number, [string, string]> => {
+  try {
+    if (!fs.existsSync(filePath)) {
+      throw new Error(`CSV file not found: ${filePath}`);
+    }
+
+    const content = fs.readFileSync(filePath, 'utf-8');
+    const lines = content.split(/\r?\n/);
+    const projectMap = new Map<number, [string, string]>();
+    let headerSkipped = false;
+
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i].trim();
+
+      if (!line || line.startsWith('#')) {
+        continue;
+      }
+
+      const values = line.split(',').map(v => v.trim());
+      const maxColumn = Math.max(idColumn, gitlabPathColumn, githubPathColumn);
+
+      if (maxColumn >= values.length) {
+        console.warn(`Warning: Line ${i + 1} has only ${values.length} column(s), skipping (need column ${maxColumn})`);
+        if (!headerSkipped) {
+          headerSkipped = true;
+        }
+        continue;
+      }
+
+      const idStr = values[idColumn];
+      const gitlabPath = values[gitlabPathColumn];
+      const githubPath = values[githubPathColumn];
+
+      if (!headerSkipped) {
+        const num = parseInt(idStr, 10);
+        if (isNaN(num) || idStr.toLowerCase().includes('id') || idStr.toLowerCase().includes('project')) {
+          console.log(`Skipping CSV header row: "${line}"`);
+          headerSkipped = true;
+          continue;
+        }
+        headerSkipped = true;
+      }
+
+      if (!idStr || !gitlabPath || !githubPath) {
+        console.warn(`Warning: Line ${i + 1} has empty values, skipping`);
+        continue;
+      }
+
+      const projectId = parseInt(idStr, 10);
+      if (isNaN(projectId)) {
+        console.warn(`Warning: Line ${i + 1}: Invalid project ID "${idStr}", skipping`);
+        continue;
+      }
+
+      projectMap.set(projectId, [gitlabPath, githubPath]);
+    }
+
+    if (projectMap.size === 0) {
+          throw new Error(`No valid project mappings found in CSV file: ${filePath}`);
+        }
+
+        console.log(`✓ Loaded ${projectMap.size} project mappings from CSV`);
+        return projectMap;
+      } catch (err) {
+        console.error(`Error reading project mapping CSV file: ${err.message}`);
+        throw err;
+  }
+};
+
 // Creates new attachments and replaces old links
 export const migrateAttachments = async (
   body: string,