| 57 | } |
| 58 | |
| 59 | func (s *Search) syncQuestionAndAnswerData(ctx context.Context, |
| 60 | syncFunc func(ctx context.Context, page, pageSize int) (answerList []*plugin.SearchContent, err error)) { |
| 61 | log.Infof("start to sync data to meilisearch") |
| 62 | for page, pageSize := 1, MaxGetPageSize; ; page++ { |
| 63 | log.Infof("start to sync page %d", page) |
| 64 | dataList, err := syncFunc(ctx, page, pageSize) |
| 65 | if err != nil { |
| 66 | log.Errorf("get data failed %s", err) |
| 67 | return |
| 68 | } |
| 69 | if len(dataList) == 0 { |
| 70 | log.Infof("get page %d success, no other data, stop sync", page) |
| 71 | break |
| 72 | } |
| 73 | log.Infof("get page %d success, record count %d", page, len(dataList)) |
| 74 | for i := 0; i < len(dataList); i += 100 { |
| 75 | end := i + MaxPutPerSize |
| 76 | if i+MaxPutPerSize > len(dataList) { |
| 77 | end = len(dataList) |
| 78 | } |
| 79 | resp, err := s.Client.Index(s.Config.IndexName).AddDocuments( |
| 80 | dataList[i:end], primaryKey) |
| 81 | if err != nil { |
| 82 | log.Errorf("add documents failed %s", err) |
| 83 | return |
| 84 | } |
| 85 | if err := waitForTask(s.Client, resp); err != nil { |
| 86 | log.Errorf("wait for task failed %s", err) |
| 87 | } |
| 88 | log.Infof("sync page %d, progress %d/%d", page, end, len(dataList)) |
| 89 | } |
| 90 | } |
| 91 | } |