From 8458e7734120195ebcfa12bf78835fd8fe7c9706 Mon Sep 17 00:00:00 2001 From: Elscrux Date: Tue, 9 Apr 2024 10:54:38 +0000 Subject: [PATCH] feat(migration): Trello organization based migration (#2211) Migrate Trello organization after organization to limit total memory allocation. Related discussion: https://community.vikunja.io/t/trello-import-issues/2110 Co-authored-by: Elscrux Co-authored-by: konrad Co-authored-by: kolaente Reviewed-on: https://kolaente.dev/vikunja/vikunja/pulls/2211 Reviewed-by: konrad Co-authored-by: Elscrux Co-committed-by: Elscrux --- pkg/modules/migration/trello/trello.go | 195 ++++++---- pkg/modules/migration/trello/trello_test.go | 384 ++++++++++++-------- 2 files changed, 364 insertions(+), 215 deletions(-) diff --git a/pkg/modules/migration/trello/trello.go b/pkg/modules/migration/trello/trello.go index b96ed35a8..b5f0558d0 100644 --- a/pkg/modules/migration/trello/trello.go +++ b/pkg/modules/migration/trello/trello.go @@ -107,80 +107,100 @@ func (m *Migration) AuthURL() string { "&return_url=" + config.MigrationTrelloRedirectURL.GetString() } -func getTrelloData(token string) (trelloData []*trello.Board, err error) { - allArg := trello.Arguments{"fields": "all"} - - client := trello.NewClient(config.MigrationTrelloKey.GetString(), token) - client.Logger = log.GetLogger() - +func getTrelloBoards(client *trello.Client) (trelloData []*trello.Board, err error) { log.Debugf("[Trello Migration] Getting boards...") trelloData, err = client.GetMyBoards(trello.Defaults()) if err != nil { - return + return nil, err } log.Debugf("[Trello Migration] Got %d trello boards", len(trelloData)) - for _, board := range trelloData { - log.Debugf("[Trello Migration] Getting projects for board %s", board.ID) + return +} - board.Lists, err = board.GetLists(trello.Defaults()) +func getTrelloOrganizationsWithBoards(boards []*trello.Board) (boardsByOrg map[string][]*trello.Board) { + + boardsByOrg = make(map[string][]*trello.Board) + + for _, board := range boards { + // Trello boards without an organization are considered personal boards + if board.IDOrganization == "" { + board.IDOrganization = "Personal" + } + + _, has := boardsByOrg[board.IDOrganization] + if !has { + boardsByOrg[board.IDOrganization] = []*trello.Board{} + } + + boardsByOrg[board.IDOrganization] = append(boardsByOrg[board.IDOrganization], board) + } + + return +} + +func fillCardData(client *trello.Client, board *trello.Board) (err error) { + allArg := trello.Arguments{"fields": "all"} + + log.Debugf("[Trello Migration] Getting projects for board %s", board.ID) + + board.Lists, err = board.GetLists(trello.Defaults()) + if err != nil { + return err + } + + log.Debugf("[Trello Migration] Got %d projects for board %s", len(board.Lists), board.ID) + + listMap := make(map[string]*trello.List, len(board.Lists)) + for _, list := range board.Lists { + listMap[list.ID] = list + } + + log.Debugf("[Trello Migration] Getting cards for board %s", board.ID) + + cards, err := board.GetCards(allArg) + if err != nil { + return + } + + log.Debugf("[Trello Migration] Got %d cards for board %s", len(cards), board.ID) + + for _, card := range cards { + list, exists := listMap[card.IDList] + if !exists { + continue + } + + card.Attachments, err = card.GetAttachments(allArg) if err != nil { return } - log.Debugf("[Trello Migration] Got %d projects for board %s", len(board.Lists), board.ID) - - listMap := make(map[string]*trello.List, len(board.Lists)) - for _, list := range board.Lists { - listMap[list.ID] = list - } - - log.Debugf("[Trello Migration] Getting cards for board %s", board.ID) - - cards, err := board.GetCards(allArg) - if err != nil { - return nil, err - } - - log.Debugf("[Trello Migration] Got %d cards for board %s", len(cards), board.ID) - - for _, card := range cards { - list, exists := listMap[card.IDList] - if !exists { - continue - } - - card.Attachments, err = card.GetAttachments(allArg) - if err != nil { - return nil, err - } - - if len(card.IDCheckLists) > 0 { - for _, checkListID := range card.IDCheckLists { - checklist, err := client.GetChecklist(checkListID, allArg) - if err != nil { - return nil, err - } - - checklist.CheckItems = []trello.CheckItem{} - err = client.Get("checklists/"+checkListID+"/checkItems", allArg, &checklist.CheckItems) - if err != nil { - return nil, err - } - - card.Checklists = append(card.Checklists, checklist) - log.Debugf("Retrieved checklist %s for card %s", checkListID, card.ID) + if len(card.IDCheckLists) > 0 { + for _, checkListID := range card.IDCheckLists { + checklist, err := client.GetChecklist(checkListID, allArg) + if err != nil { + return err } - } - list.Cards = append(list.Cards, card) + checklist.CheckItems = []trello.CheckItem{} + err = client.Get("checklists/"+checkListID+"/checkItems", allArg, &checklist.CheckItems) + if err != nil { + return err + } + + card.Checklists = append(card.Checklists, checklist) + log.Debugf("Retrieved checklist %s for card %s", checkListID, card.ID) + } } - log.Debugf("[Trello Migration] Looked for attachements on all cards of board %s", board.ID) + list.Cards = append(list.Cards, card) } + log.Debugf("[Trello Migration] Looked for attachements on all cards of board %s", board.ID) + return } @@ -196,7 +216,7 @@ func convertMarkdownToHTML(input string) (output string, err error) { // Converts all previously obtained data from trello into the vikunja format. // `trelloData` should contain all boards with their projects and cards respectively. -func convertTrelloDataToVikunja(trelloData []*trello.Board, token string) (fullVikunjaHierachie []*models.ProjectWithTasksAndBuckets, err error) { +func convertTrelloDataToVikunja(organizationName string, trelloData []*trello.Board, token string) (fullVikunjaHierachie []*models.ProjectWithTasksAndBuckets, err error) { log.Debugf("[Trello Migration] ") @@ -205,7 +225,7 @@ func convertTrelloDataToVikunja(trelloData []*trello.Board, token string) (fullV { Project: models.Project{ ID: pseudoParentID, - Title: "Imported from Trello", + Title: organizationName, }, }, } @@ -392,29 +412,58 @@ func (m *Migration) Migrate(u *user.User) (err error) { log.Debugf("[Trello Migration] Starting migration for user %d", u.ID) log.Debugf("[Trello Migration] Getting all trello data for user %d", u.ID) - trelloData, err := getTrelloData(m.Token) + client := trello.NewClient(config.MigrationTrelloKey.GetString(), m.Token) + client.Logger = log.GetLogger() + + boards, err := getTrelloBoards(client) if err != nil { return } log.Debugf("[Trello Migration] Got all trello data for user %d", u.ID) - log.Debugf("[Trello Migration] Start converting trello data for user %d", u.ID) - fullVikunjaHierachie, err := convertTrelloDataToVikunja(trelloData, m.Token) - if err != nil { - return + organizationMap := getTrelloOrganizationsWithBoards(boards) + for organizationID, boards := range organizationMap { + log.Debugf("[Trello Migration] Getting organization with id %s for user %d", organizationID, u.ID) + orgName := organizationID + if organizationID != "Personal" { + organization, err := client.GetOrganization(organizationID, trello.Defaults()) + if err != nil { + return err + } + orgName = organization.DisplayName + } + + for _, board := range boards { + log.Debugf("[Trello Migration] Getting card data for board %s for user %d for organization %s", board.ID, u.ID, organizationID) + + err = fillCardData(client, board) + if err != nil { + return err + } + + log.Debugf("[Trello Migration] Got card data for board %s for user %d for organization %s", board.ID, u.ID, organizationID) + } + + log.Debugf("[Trello Migration] Start converting trello data for user %d for organization %s", u.ID, organizationID) + + hierarchy, err := convertTrelloDataToVikunja(orgName, boards, m.Token) + if err != nil { + return err + } + + log.Debugf("[Trello Migration] Done migrating trello data for user %d for organization %s", u.ID, organizationID) + log.Debugf("[Trello Migration] Start inserting trello data for user %d for organization %s", u.ID, organizationID) + + err = migration.InsertFromStructure(hierarchy, u) + if err != nil { + return err + } + + log.Debugf("[Trello Migration] Done inserting trello data for user %d for organization %s", u.ID, organizationID) } - log.Debugf("[Trello Migration] Done migrating trello data for user %d", u.ID) - log.Debugf("[Trello Migration] Start inserting trello data for user %d", u.ID) + log.Debugf("[Trello Migration] Done migrating all trello data for user %d", u.ID) - err = migration.InsertFromStructure(fullVikunjaHierachie, u) - if err != nil { - return - } - - log.Debugf("[Trello Migration] Done inserting trello data for user %d", u.ID) - log.Debugf("[Trello Migration] Migration done for user %d", u.ID) - - return nil + return } diff --git a/pkg/modules/migration/trello/trello_test.go b/pkg/modules/migration/trello/trello_test.go index 4099a2250..9c7bc63ac 100644 --- a/pkg/modules/migration/trello/trello_test.go +++ b/pkg/modules/migration/trello/trello_test.go @@ -32,20 +32,23 @@ import ( "github.com/stretchr/testify/require" ) -func TestConvertTrelloToVikunja(t *testing.T) { +func getTestBoard(t *testing.T) ([]*trello.Board, time.Time) { config.InitConfig() time1, err := time.Parse(time.RFC3339Nano, "2014-09-26T08:25:05Z") require.NoError(t, err) - exampleFile, err := os.ReadFile(config.ServiceRootpath.GetString() + "/pkg/modules/migration/testimage.jpg") - require.NoError(t, err) trelloData := []*trello.Board{ { - Name: "TestBoard", - Desc: "This is a description", - Closed: false, + Name: "TestBoard", + Organization: trello.Organization{ + ID: "orgid", + DisplayName: "TestOrg", + }, + IDOrganization: "orgid", + Desc: "This is a description", + Closed: false, Lists: []*trello.List{ { Name: "Test Project 1", @@ -168,8 +171,13 @@ func TestConvertTrelloToVikunja(t *testing.T) { }, }, { - Name: "TestBoard 2", - Closed: false, + Organization: trello.Organization{ + ID: "orgid2", + DisplayName: "TestOrg2", + }, + IDOrganization: "orgid2", + Name: "TestBoard 2", + Closed: false, Lists: []*trello.List{ { Name: "Test Project 4", @@ -183,8 +191,13 @@ func TestConvertTrelloToVikunja(t *testing.T) { }, }, { - Name: "TestBoard Archived", - Closed: true, + Organization: trello.Organization{ + ID: "orgid", + DisplayName: "TestOrg", + }, + IDOrganization: "orgid", + Name: "TestBoard Archived", + Closed: true, Lists: []*trello.List{ { Name: "Test Project 5", @@ -197,67 +210,91 @@ func TestConvertTrelloToVikunja(t *testing.T) { }, }, }, + { + Name: "Personal Board", + Lists: []*trello.List{ + { + Name: "Test Project 6", + Cards: []*trello.Card{ + { + Name: "Test Card 5659", + Pos: 123, + }, + }, + }, + }, + }, } trelloData[0].Prefs.BackgroundImage = "https://vikunja.io/testimage.jpg" // Using an image which we are hosting, so it'll still be up - expectedHierachie := []*models.ProjectWithTasksAndBuckets{ - { - Project: models.Project{ - ID: 1, - Title: "Imported from Trello", - }, - }, - { - Project: models.Project{ - ID: 2, - ParentProjectID: 1, - Title: "TestBoard", - Description: "This is a description", - BackgroundInformation: bytes.NewBuffer(exampleFile), - }, - Buckets: []*models.Bucket{ - { + return trelloData, time1 +} + +func TestConvertTrelloToVikunja(t *testing.T) { + trelloData, time1 := getTestBoard(t) + + exampleFile, err := os.ReadFile(config.ServiceRootpath.GetString() + "/pkg/modules/migration/testimage.jpg") + require.NoError(t, err) + + expectedHierarchyOrg := map[string][]*models.ProjectWithTasksAndBuckets{ + "orgid": { + { + Project: models.Project{ ID: 1, - Title: "Test Project 1", - }, - { - ID: 2, - Title: "Test Project 2", + Title: "orgid", }, }, - Tasks: []*models.TaskWithComments{ - { - Task: models.Task{ - Title: "Test Card 1", - Description: "

Card Description bold

\n", - BucketID: 1, - DueDate: time1, - Labels: []*models.Label{ - { - Title: "Label 1", - HexColor: trelloColorMap["green"], + { + Project: models.Project{ + ID: 2, + ParentProjectID: 1, + Title: "TestBoard", + Description: "This is a description", + BackgroundInformation: bytes.NewBuffer(exampleFile), + }, + Buckets: []*models.Bucket{ + { + ID: 1, + Title: "Test Project 1", + }, + { + ID: 2, + Title: "Test Project 2", + }, + }, + Tasks: []*models.TaskWithComments{ + { + Task: models.Task{ + Title: "Test Card 1", + Description: "

Card Description bold

\n", + BucketID: 1, + DueDate: time1, + Labels: []*models.Label{ + { + Title: "Label 1", + HexColor: trelloColorMap["green"], + }, + { + Title: "Label 2", + HexColor: trelloColorMap["orange"], + }, }, - { - Title: "Label 2", - HexColor: trelloColorMap["orange"], - }, - }, - Attachments: []*models.TaskAttachment{ - { - File: &files.File{ - Name: "Testimage.jpg", - Mime: "image/jpg", - Size: uint64(len(exampleFile)), - FileContent: exampleFile, + Attachments: []*models.TaskAttachment{ + { + File: &files.File{ + Name: "Testimage.jpg", + Mime: "image/jpg", + Size: uint64(len(exampleFile)), + FileContent: exampleFile, + }, }, }, }, }, - }, - { - Task: models.Task{ - Title: "Test Card 2", - Description: ` + { + Task: models.Task{ + Title: "Test Card 2", + Description: `

Checkproject 1

@@ -270,117 +307,180 @@ func TestConvertTrelloToVikunja(t *testing.T) {

  • Pending Task

  • Another Pending Task

`, - BucketID: 1, + BucketID: 1, + }, }, - }, - { - Task: models.Task{ - Title: "Test Card 3", - BucketID: 1, + { + Task: models.Task{ + Title: "Test Card 3", + BucketID: 1, + }, }, - }, - { - Task: models.Task{ - Title: "Test Card 4", - BucketID: 1, - Labels: []*models.Label{ - { - Title: "Label 2", - HexColor: trelloColorMap["orange"], + { + Task: models.Task{ + Title: "Test Card 4", + BucketID: 1, + Labels: []*models.Label{ + { + Title: "Label 2", + HexColor: trelloColorMap["orange"], + }, }, }, }, - }, - { - Task: models.Task{ - Title: "Test Card 5", - BucketID: 2, - Labels: []*models.Label{ - { - Title: "Label 3", - HexColor: trelloColorMap["blue"], - }, - { - Title: "Label 4", - HexColor: trelloColorMap["green_dark"], - }, - { - Title: "Label 5", - HexColor: trelloColorMap["transparent"], + { + Task: models.Task{ + Title: "Test Card 5", + BucketID: 2, + Labels: []*models.Label{ + { + Title: "Label 3", + HexColor: trelloColorMap["blue"], + }, + { + Title: "Label 4", + HexColor: trelloColorMap["green_dark"], + }, + { + Title: "Label 5", + HexColor: trelloColorMap["transparent"], + }, }, }, }, - }, - { - Task: models.Task{ - Title: "Test Card 6", - BucketID: 2, - DueDate: time1, + { + Task: models.Task{ + Title: "Test Card 6", + BucketID: 2, + DueDate: time1, + }, + }, + { + Task: models.Task{ + Title: "Test Card 7", + BucketID: 2, + }, + }, + { + Task: models.Task{ + Title: "Test Card 8", + BucketID: 2, + }, }, }, - { - Task: models.Task{ - Title: "Test Card 7", - BucketID: 2, + }, + { + Project: models.Project{ + ID: 3, + ParentProjectID: 1, + Title: "TestBoard Archived", + IsArchived: true, + }, + Buckets: []*models.Bucket{ + { + ID: 3, + Title: "Test Project 5", }, }, - { - Task: models.Task{ - Title: "Test Card 8", - BucketID: 2, + Tasks: []*models.TaskWithComments{ + { + Task: models.Task{ + Title: "Test Card 63423", + BucketID: 3, + }, }, }, }, }, - { - Project: models.Project{ - ID: 3, - ParentProjectID: 1, - Title: "TestBoard 2", - }, - Buckets: []*models.Bucket{ - { - ID: 3, - Title: "Test Project 4", + "orgid2": { + { + Project: models.Project{ + ID: 1, + Title: "orgid2", }, }, - Tasks: []*models.TaskWithComments{ - { - Task: models.Task{ - Title: "Test Card 634", - BucketID: 3, + { + Project: models.Project{ + ID: 2, + ParentProjectID: 1, + Title: "TestBoard 2", + }, + Buckets: []*models.Bucket{ + { + ID: 1, + Title: "Test Project 4", + }, + }, + Tasks: []*models.TaskWithComments{ + { + Task: models.Task{ + Title: "Test Card 634", + BucketID: 1, + }, }, }, }, }, - { - Project: models.Project{ - ID: 4, - ParentProjectID: 1, - Title: "TestBoard Archived", - IsArchived: true, - }, - Buckets: []*models.Bucket{ - { - ID: 4, - Title: "Test Project 5", + "Personal": { + { + Project: models.Project{ + ID: 1, + Title: "Personal", }, }, - Tasks: []*models.TaskWithComments{ - { - Task: models.Task{ - Title: "Test Card 63423", - BucketID: 4, + { + Project: models.Project{ + ID: 2, + ParentProjectID: 1, + Title: "Personal Board", + }, + Buckets: []*models.Bucket{ + { + ID: 1, + Title: "Test Project 6", + }, + }, + Tasks: []*models.TaskWithComments{ + { + Task: models.Task{ + Title: "Test Card 5659", + BucketID: 1, + }, }, }, }, }, } - hierachie, err := convertTrelloDataToVikunja(trelloData, "") - require.NoError(t, err) - assert.NotNil(t, hierachie) - if diff, equal := messagediff.PrettyDiff(hierachie, expectedHierachie); !equal { - t.Errorf("converted trello data = %v, want %v, diff: %v", hierachie, expectedHierachie, diff) + organizationMap := getTrelloOrganizationsWithBoards(trelloData) + for organizationID, boards := range organizationMap { + hierarchy, err := convertTrelloDataToVikunja(organizationID, boards, "") + + require.NoError(t, err) + assert.NotNil(t, hierarchy) + if diff, equal := messagediff.PrettyDiff(hierarchy, expectedHierarchyOrg[organizationID]); !equal { + t.Errorf("converted trello data = %v,\nwant %v,\ndiff: %v", hierarchy, expectedHierarchyOrg[organizationID], diff) + } + } +} + +func TestCreateOrganizationMap(t *testing.T) { + trelloData, _ := getTestBoard(t) + + organizationMap := getTrelloOrganizationsWithBoards(trelloData) + expectedMap := map[string][]*trello.Board{ + "orgid": { + trelloData[0], + trelloData[2], + }, + "orgid2": { + trelloData[1], + }, + "Personal": { + trelloData[3], + }, + } + if diff, equal := messagediff.PrettyDiff(organizationMap, expectedMap); !equal { + t.Errorf("converted trello data = %v,\nwant %v,\ndiff: %v", organizationMap, expectedMap, diff) } }