lakedrops/portal/data-core/datacore-parent#30 Add DataCore tasks for scheduled pipelines

datacore.yml

 stages:
   - build
   - prepare
+  - postprocess
   - deploy
 
 include:
@@ -8,6 +9,9 @@ include:
     ref: main
     file: '/mixins.yml'
 
+variables:
+  DATACORE_IMPORT_INTERNAL_USERS: 'no'
+
 DeployDataCore:
   stage: build
   tags:
@@ -67,6 +71,9 @@ DeployDataCore:
     refs:
       - develop
       - /^feature.*$/
+  except:
+    refs:
+      - schedule
 
 Validate Export:
   stage: build
@@ -94,8 +101,10 @@ Validate Export:
   except:
     variables:
       - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'main'
+    refs:
+      - schedule
 
-ImportIntoDatacore:
+Prepare Import:
   stage: prepare
   tags:
     - deploylocal
@@ -108,19 +117,28 @@ ImportIntoDatacore:
     - git fetch
     - git pull
     - cd /data/${CI_COMMIT_REF_NAME}
+  variables:
+    GIT_STRATEGY: none
+  cache: {}
+  only:
+    refs:
+      - develop
+      - /^feature.*$/
+      - schedules
+
+ImportIntoDatacore:
+  stage: prepare
+  tags:
+    - deploylocal
+  needs:
+    - job: PrepareImport
+  before_script: !reference [.prepareaccess, before_script]
+  script:
     - dc exec app curl --fail -X POST http://localhost:4100/api/import/basic
     - dc exec app curl --fail -X POST http://localhost:4100/api/import/realestate
+    - if [[ "DATACORE_IMPORT_INTERNAL_USERS" == "yes ]]; then dc exec app curl --fail -X POST http://localhost:4100/api/import/internalusers; fi
     - dc exec app curl --fail -X GET http://localhost:4100/api/export/basic >/dev/null 2>&1
     - dc exec app curl --fail -X GET http://localhost:4100/api/export/realestate >/dev/null 2>&1
-    - cd /data/${CI_COMMIT_REF_NAME}/data
-    - git status > /tmp/gitstatus.log
-    - EC=0
-    - grep "nothing to commit, working tree clean" /tmp/gitstatus.log || EC=$?
-    - if [[ $EC -eq 0 ]]; then exit 0; fi
-    - git add *
-    - git commit -am "Updated assets by data core"
-    - if [[ "$CI_COMMIT_REF_NAME" == "develop" ]]; then git push -o ci.skip -o merge_request.create -o merge_request.target=main -o merge_request.merge_when_pipeline_succeeds; fi
-    - if [[ "$CI_COMMIT_REF_NAME" != "develop" ]]; then git push -o ci.skip; fi
   variables:
     GIT_STRATEGY: none
   cache: {}
@@ -128,6 +146,9 @@ ImportIntoDatacore:
     refs:
       - develop
       - /^feature.*$/
+  except:
+    refs:
+      - schedule
 
 .deployToDrupal:
   stage: deploy
@@ -140,10 +161,11 @@ ImportIntoDatacore:
   except:
     refs:
       - tags
+      - schedule
   script:
     - curl -X POST --fail -F token=${TRIGGERTOKEN} -F ref=${CI_COMMIT_REF_NAME} -F "variables[TRIGGERTASK]=DeployAssetsToDrupal" https://gitlab.lakedrops.com/api/v4/projects/${TRIGGERPROJECTID}/trigger/pipeline
 
-DataCore XRM Check New Users:
+XRM Check New Users:
   stage: build
   tags:
     - deploylocal
@@ -152,23 +174,16 @@ DataCore XRM Check New Users:
   cache: {}
   only:
     refs:
-      - chat
       - schedules
   except:
     variables:
       - $DATACORE_JOB_NAME && $DATACORE_JOB_NAME != "XrmCheckNewUser"
   before_script: !reference [.prepareaccess, before_script]
   script:
-    - if [[ ! -d /data/develop/data/.git ]]; then git clone -b develop "git@$CI_SERVER_HOST:$CI_PROJECT_PATH.git" /data/develop/data; fi
-    - cd /data/develop/data
-    - git config user.email "deployment@lakedrops.com"
-    - git config user.name "Deployment"
-    - git fetch
-    - git pull
-    - cd /data/develop
+    - cd /data/${CI_COMMIT_REF_NAME}
     - dc exec app curl --fail -X GET http://localhost:4000/api/import/realestate/xrm/newusers >/dev/null 2>&1
 
-DataCore XRM Sync:
+XRM Sync:
   stage: build
   tags:
     - deploylocal
@@ -177,18 +192,42 @@ DataCore XRM Sync:
   cache: {}
   only:
     refs:
-      - chat
       - schedules
   except:
     variables:
       - $DATACORE_JOB_NAME && $DATACORE_JOB_NAME != "XrmSync"
   before_script: !reference [.prepareaccess, before_script]
   script:
-    - if [[ ! -d /data/develop/data/.git ]]; then git clone -b develop "git@$CI_SERVER_HOST:$CI_PROJECT_PATH.git" /data/develop/data; fi
-    - cd /data/develop/data
-    - git config user.email "deployment@lakedrops.com"
-    - git config user.name "Deployment"
-    - git fetch
-    - git pull
-    - cd /data/develop
+    - cd /data/${CI_COMMIT_REF_NAME}
     - dc exec app curl --fail -X GET http://localhost:4000/api/import/realestate/xrm/sync >/dev/null 2>&1
+
+Push Changes:
+  stage: postprocess
+  tags:
+    - deploylocal
+  needs:
+    - job: ImportIntoDatacore
+      optional: true
+    - job: XRM Check New Users
+      optional: true
+    - job: XRM Sync
+      optional: true
+  before_script: !reference [.prepareaccess, before_script]
+  script:
+    - cd /data/${CI_COMMIT_REF_NAME}/data
+    - git status > /tmp/gitstatus.log
+    - EC=0
+    - grep "nothing to commit, working tree clean" /tmp/gitstatus.log || EC=$?
+    - if [[ $EC -eq 0 ]]; then exit 0; fi
+    - git add *
+    - git commit -am "Updated assets by data core"
+    - if [[ "$CI_COMMIT_REF_NAME" == "develop" ]]; then git push -o ci.skip -o merge_request.create -o merge_request.target=main -o merge_request.merge_when_pipeline_succeeds; fi
+    - if [[ "$CI_COMMIT_REF_NAME" != "develop" ]]; then git push -o ci.skip; fi
+  variables:
+    GIT_STRATEGY: none
+  cache: {}
+  only:
+    refs:
+      - develop
+      - /^feature.*$/
+      - schedules