Enable execution on CircleCI for more notebooks

ignore_tutorials/ignore_circleci_testing

@@ -1,13 +1,7 @@
-# ignore these 10 now as excluding execution for these is a combination that passes JB2 building on CircleCI.
+# ignore these as they run out of resources on CircleCI.
 # Note that the first 2 are excluded for all kinds of testing; while the parallelize causes issues for GHA JB2 builds and non linux testing, too.
 #
-tutorials/parquet-catalog-demos/neowise-source-table-lightcurves
-tutorials/parquet-catalog-demos/neowise-source-table-strategies
-tutorials/parallelize/Parallelize_Convolution
 tutorials/roman_simulations/roman_hlss_number_density
-tutorials/cosmodc2/cosmoDC2_TAP_access
 tutorials/openuniversesims/openuniverse2024_roman_simulated_timedomainsurvey
-tutorials/openuniversesims/openuniverse2024_roman_simulated_wideareasurvey
-tutorials/parquet-catalog-demos/wise-allwise-catalog-demo
 tutorials/cloud_access/euclid-cloud-access
 tutorials/cloud_access/cloud-access-intro

tox.ini

@@ -68,24 +68,24 @@ commands =
     bash -c 'find tutorials -name "*md" | xargs grep kernelspec: | awk -F :kernelspec: "{print \$1}" > all_tutorials'
 
     # Make a list of the tutorials changed, we only need this in CI. Also deal with grep's non-zero exit code.
-    bash -c 'if [[ $CI == true ]]; then git fetch origin main --depth=1; git diff origin/main --name-only tutorials | grep ".md" || true; fi > changed_tutorials'
-
+    #bash -c 'if [[ $CI == true ]]; then git fetch origin main --depth=1; git diff origin/main --name-only tutorials | (grep ".md" || true); fi > changed_tutorials'
+    bash -c 'cat all_tutorials > changed_tutorials'
     # We only skip testing untouched tutorials in PRs; in cron and dispatch all should be tested
     !buildhtml: bash -c 'if [[ $GITHUB_EVENT_NAME == pull_request && -z "$(grep force_run:all_tests ${GITHUB_EVENT_PATH})" ]]; then cat changed_tutorials; else cat all_tutorials; fi | grep -vf ignore_testing | xargs jupytext --to notebook '
 
-    !buildhtml: bash -c "echo 'Notebooks ignored (not tested/executed) in this job:\n'; cat ignore_testing"
+    !buildhtml: bash -c "echo 'Notebooks ignored (not tested/executed) in this job:'; cat ignore_testing"
     !buildhtml: pytest --nbval-lax -vv --suppress-no-test-exit-code --durations=10 tutorials
 
     # We don't want to execute in rendering the ones we ignore for testing
-    buildhtml: bash -c 'grep -f ignore_testing all_tutorials > ignore_execute'
+    buildhtml: bash -c '(grep -f ignore_testing all_tutorials || true) > ignore_execute'
 
     # On CircleCI rendering preview, we also don't want to execute anything that hasn't been modified
-    buildhtml: bash -c 'if [[ $CIRCLECI == true ]]; then grep -vf changed_tutorials all_tutorials >> ignore_execute; fi'
+    buildhtml: bash -c 'if [[ $CIRCLECI == true ]]; then (grep -vf changed_tutorials all_tutorials || true) >> ignore_execute; fi'
 
     # sed -i needs a bit of hacky conditional on ubuntu to cover the case of an empty ignore
     buildhtml: bash -c 'if [ -s ignore_execute ]; then for name in $(cat ignore_execute | sort| uniq); do if [ -z "$(head -n 20 ${name}| grep execute:)" ]; then sed -i -e "s|kernelspec:|execute:\n  skip: true\nkernelspec:|g" ${name}; fi; done;fi'
 
-    buildhtml: bash -c "echo 'Notebooks ignored (not tested/executed) in this job:\n'; cat ignore_execute"
+    buildhtml: bash -c "echo 'Notebooks ignored (not tested/executed) in this job:'; cat ignore_execute"
 
     # Using srtict so we fail with trackbacks and debug mode to have a richer log
     # For full build we disable parallel runs to easy server load