<core_client_version>7.16.6</core_client_version>
<![CDATA[
<message>
process exited with code 1 (0x1, -255)</message>
<stderr_txt>
v7-Notebook-PC:1:1:968648:968648] [20221222:233515:1671748515:0.000] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:968648:968648] [20221222:233515:1671748515:0.000] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:968648:968648] [20221222:233515:1671748515:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:968648:968648] [20221222:233515:1671748515:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:968648:968648] [20221222:233515:1671748515:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:968648:968648] [20221222:233515:1671748515:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:968648:968648] [20221222:233515:1671748515:0.000] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:968648:968648] [20221222:233515:1671748515:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:968648:968648] [20221222:233515:1671748515:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:968648:968648] [20221222:233515:1671748515:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:968648:968648] [20221222:233515:1671748515:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:968648:968648] [20221222:233515:1671748515:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:968648:968648] [20221222:233515:1671748515:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:968648:968648] [20221222:233515:1671748515:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:968648:968648] [20221222:233515:1671748515:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at (nil))
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:968648:968648] [20221222:233515:1671748515:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:968648:968648] [20221222:233515:1671748515:0.001] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
23:36:38 STEP 240 H= 240:00 +CPU= 59.260
23:37:12 STEP 241 H= 241:00 +CPU= 33.312
23:37:44 STEP 242 H= 242:00 +CPU= 32.675
23:38:36 STEP 243 H= 243:00 +CPU= 51.856
23:39:10 STEP 244 H= 244:00 +CPU= 33.527
23:39:43 STEP 245 H= 245:00 +CPU= 32.897
23:40:38 STEP 246 H= 246:00 +CPU= 53.772
23:41:10 STEP 247 H= 247:00 +CPU= 32.743
23:41:45 STEP 248 H= 248:00 +CPU= 34.602
23:42:37 STEP 249 H= 249:00 +CPU= 51.988
23:43:10 STEP 250 H= 250:00 +CPU= 32.809
23:43:41 STEP 251 H= 251:00 +CPU= 30.819
23:44:40 STEP 252 H= 252:00 +CPU= 58.809
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhq0f+000252
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhq0f+000252
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhq0f+000252
23:45:12 STEP 253 H= 253:00 +CPU= 31.349
23:45:57 STEP 254 H= 254:00 +CPU= 44.554
23:46:44 STEP 255 H= 255:00 +CPU= 47.664
23:47:18 STEP 256 H= 256:00 +CPU= 33.415
23:47:48 STEP 257 H= 257:00 +CPU= 29.905
23:48:44 STEP 258 H= 258:00 +CPU= 55.356
23:49:15 STEP 259 H= 259:00 +CPU= 30.841
23:49:51 STEP 260 H= 260:00 +CPU= 35.870
23:50:40 STEP 261 H= 261:00 +CPU= 49.080
23:51:17 STEP 262 H= 262:00 +CPU= 36.622
23:51:48 STEP 263 H= 263:00 +CPU= 30.927
23:53:40 STEP 264 H= 264:00 +CPU= 63.029
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhq0f+000264
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhq0f+000264
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhq0f+000264
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMGGhq0f+000240
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMSHhq0f+000240
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMUAhq0f+000240
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMGGhq0f+000252
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMSHhq0f+000252
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMUAhq0f+000252
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0977_1982050100_123_951_12168621_0_r1802458006_10.zip
Uploading the intermediate file: upload_file_10.zip
23:54:58 STEP 265 H= 265:00 +CPU= 30.297
23:55:39 STEP 266 H= 266:00 +CPU= 28.532
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
Suspend request received from the BOINC client, suspending the child process
Quit request received from the BOINC client, ending the child process
(argv0) ../../projects/climateprediction.net/oifs_43r3_ps_1.05_x86_64-pc-linux-gnu
(argv1) start_date: 1982050100
(argv2) exptid: hq0f
(argv3) unique_member_id: 0977
(argv4) batchid: 951
(argv5) wuid: 12168621
(argv6) fclen: 123
(argv7) app_name: oifs_43r3_ps
(argv8) nthreads: 1
Working directory is: /var/lib/boinc-client/slots/3
Project directory is: /var/lib/boinc-client/projects/climateprediction.net/
app name: oifs_43r3_ps
version: 1.05
Location of temp folder: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621
..mkdir for temp folder for results failed
Copying: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip to: /var/lib/boinc-client/slots/3/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc-client/slots/3/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_baad597788e0abfd41ee1eaf1ab4f9da to: /var/lib/boinc-client/slots/3/oifs_43r3_ps_0977_1982050100_123_951_12168621.zip
Unzipping the namelist zip file: /var/lib/boinc-client/slots/3/oifs_43r3_ps_0977_1982050100_123_951_12168621.zip
ic_ancil_file: ic_ancil_12168621
ifsdata_file: ifsdata_12168621
climate_data_file: clim_data_12168621
horiz_resolution: 159
vert_resolution: 60
grid_type: l_2
upload_interval: 24
utstep: 3600
nfrres: restart dump frequency (steps) 24
Copying IC ancils from: ../../projects/climateprediction.net/jf_eb65d99f44920123031e01150753b734 to: /var/lib/boinc-client/slots/3/ic_ancil_12168621.zip
Unzipping the IC ancils zip file: /var/lib/boinc-client/slots/3/ic_ancil_12168621.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_91ea09656a4143e55437ce21e7741399 to: /var/lib/boinc-client/slots/3/ifsdata/ifsdata_12168621.zip
Unzipping the ifsdata_zip file: /var/lib/boinc-client/slots/3/ifsdata/ifsdata_12168621.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc-client/slots/3/159l_2/clim_data_12168621.zip
Unzipping the climate data zip file: /var/lib/boinc-client/slots/3/159l_2/clim_data_12168621.zip
Checking for progress XML file: /var/lib/boinc-client/slots/3/progress_file_12168621.xml
Opened progress file ok : /var/lib/boinc-client/slots/3/progress_file_12168621.xml
-- Model is restarting --
Adjusting last_iter, 267, to previous model restart step.
Creating progress file: /var/lib/boinc-client/slots/3/progress_file_12168621.xml
last_cpu_time: 11762
upload_file_number: 11
last_iter: 263
last_upload: 950400
model_completed: 0
total_length_of_simulation: 10627200
result_base_name: oifs_43r3_ps_0977_1982050100_123_951_12168621_0_r1802458006
The child process has been launched with process id: 972560
Executing the command: /var/lib/boinc-client/slots/3/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2091] fp = 0x3129fe0
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at (nil))
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:972560:972560] [20221223:010136:1671753696:0.001] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
01:03:58 STEP 264 H= 264:00 +CPU= 59.816
01:04:30 STEP 265 H= 265:00 +CPU= 31.532
01:05:02 STEP 266 H= 266:00 +CPU= 31.976
01:05:53 STEP 267 H= 267:00 +CPU= 50.185
01:06:26 STEP 268 H= 268:00 +CPU= 32.118
01:06:56 STEP 269 H= 269:00 +CPU= 30.381
01:07:48 STEP 270 H= 270:00 +CPU= 50.990
01:08:20 STEP 271 H= 271:00 +CPU= 31.667
01:08:54 STEP 272 H= 272:00 +CPU= 34.238
01:09:45 STEP 273 H= 273:00 +CPU= 50.060
01:10:22 STEP 274 H= 274:00 +CPU= 34.793
01:10:53 STEP 275 H= 275:00 +CPU= 31.159
01:11:51 STEP 276 H= 276:00 +CPU= 56.554
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhq0f+000276
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhq0f+000276
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhq0f+000276
01:12:22 STEP 277 H= 277:00 +CPU= 31.571
01:12:55 STEP 278 H= 278:00 +CPU= 32.847
01:13:47 STEP 279 H= 279:00 +CPU= 51.023
01:14:20 STEP 280 H= 280:00 +CPU= 32.842
01:14:53 STEP 281 H= 281:00 +CPU= 32.503
01:15:45 STEP 282 H= 282:00 +CPU= 51.205
01:16:16 STEP 283 H= 283:00 +CPU= 31.183
01:16:50 STEP 284 H= 284:00 +CPU= 33.666
01:17:39 STEP 285 H= 285:00 +CPU= 48.310
01:18:12 STEP 286 H= 286:00 +CPU= 33.247
01:18:46 STEP 287 H= 287:00 +CPU= 32.450
01:19:36 STEP 288 H= 288:00 +CPU= 45.497
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhq0f+000288
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhq0f+000288
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhq0f+000288
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMGGhq0f+000264
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMSHhq0f+000264
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMUAhq0f+000264
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMGGhq0f+000276
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMSHhq0f+000276
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMUAhq0f+000276
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0977_1982050100_123_951_12168621_0_r1802458006_11.zip
Uploading the intermediate file: upload_file_11.zip
01:20:08 STEP 289 H= 289:00 +CPU= 31.484
01:20:41 STEP 290 H= 290:00 +CPU= 32.933
01:21:31 STEP 291 H= 291:00 +CPU= 49.960
01:22:05 STEP 292 H= 292:00 +CPU= 33.093
01:22:37 STEP 293 H= 293:00 +CPU= 32.287
01:23:29 STEP 294 H= 294:00 +CPU= 51.882
01:24:01 STEP 295 H= 295:00 +CPU= 31.336
01:24:34 STEP 296 H= 296:00 +CPU= 33.240
01:25:25 STEP 297 H= 297:00 +CPU= 50.539
01:25:58 STEP 298 H= 298:00 +CPU= 33.332
01:26:31 STEP 299 H= 299:00 +CPU= 32.394
01:27:31 STEP 300 H= 300:00 +CPU= 56.176
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhq0f+000300
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhq0f+000300
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhq0f+000300
01:28:04 STEP 301 H= 301:00 +CPU= 31.967
01:28:36 STEP 302 H= 302:00 +CPU= 31.806
01:29:26 STEP 303 H= 303:00 +CPU= 50.146
01:29:59 STEP 304 H= 304:00 +CPU= 32.581
01:30:31 STEP 305 H= 305:00 +CPU= 32.359
01:31:25 STEP 306 H= 306:00 +CPU= 52.324
01:31:57 STEP 307 H= 307:00 +CPU= 31.561
01:32:30 STEP 308 H= 308:00 +CPU= 32.560
01:33:18 STEP 309 H= 309:00 +CPU= 48.269
01:33:51 STEP 310 H= 310:00 +CPU= 32.852
01:34:27 STEP 311 H= 311:00 +CPU= 34.281
01:35:28 STEP 312 H= 312:00 +CPU= 55.668
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhq0f+000312
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhq0f+000312
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhq0f+000312
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMGGhq0f+000288
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMSHhq0f+000288
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMUAhq0f+000288
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMGGhq0f+000300
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMSHhq0f+000300
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMUAhq0f+000300
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0977_1982050100_123_951_12168621_0_r1802458006_12.zip
Uploading the intermediate file: upload_file_12.zip
01:35:59 STEP 313 H= 313:00 +CPU= 31.413
01:36:32 STEP 314 H= 314:00 +CPU= 32.293
01:37:23 STEP 315 H= 315:00 +CPU= 50.601
01:37:54 STEP 316 H= 316:00 +CPU= 31.335
01:38:26 STEP 317 H= 317:00 +CPU= 31.445
01:39:18 STEP 318 H= 318:00 +CPU= 52.302
01:39:51 STEP 319 H= 319:00 +CPU= 32.160
01:40:25 STEP 320 H= 320:00 +CPU= 34.153
01:41:16 STEP 321 H= 321:00 +CPU= 51.075
01:41:50 STEP 322 H= 322:00 +CPU= 33.171
01:42:23 STEP 323 H= 323:00 +CPU= 33.187
01:43:19 STEP 324 H= 324:00 +CPU= 54.699
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhq0f+000324
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhq0f+000324
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhq0f+000324
01:43:52 STEP 325 H= 325:00 +CPU= 32.797
01:44:25 STEP 326 H= 326:00 +CPU= 32.395
01:45:16 STEP 327 H= 327:00 +CPU= 51.139
01:45:50 STEP 328 H= 328:00 +CPU= 33.626
01:46:20 STEP 329 H= 329:00 +CPU= 30.454
01:47:13 STEP 330 H= 330:00 +CPU= 52.380
01:47:44 STEP 331 H= 331:00 +CPU= 31.331
01:48:18 STEP 332 H= 332:00 +CPU= 33.080
01:49:08 STEP 333 H= 333:00 +CPU= 50.792
01:49:40 STEP 334 H= 334:00 +CPU= 31.719
01:50:15 STEP 335 H= 335:00 +CPU= 33.204
01:51:15 STEP 336 H= 336:00 +CPU= 57.485
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhq0f+000336
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhq0f+000336
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhq0f+000336
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMGGhq0f+000312
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMSHhq0f+000312
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMUAhq0f+000312
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMGGhq0f+000324
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMSHhq0f+000324
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMUAhq0f+000324
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0977_1982050100_123_951_12168621_0_r1802458006_13.zip
Uploading the intermediate file: upload_file_13.zip
01:51:43 STEP 337 H= 337:00 +CPU= 28.066
01:52:16 STEP 338 H= 338:00 +CPU= 32.912
01:53:07 STEP 339 H= 339:00 +CPU= 50.184
01:53:38 STEP 340 H= 340:00 +CPU= 31.403
01:54:11 STEP 341 H= 341:00 +CPU= 32.231
01:55:02 STEP 342 H= 342:00 +CPU= 51.073
01:55:33 STEP 343 H= 343:00 +CPU= 31.027
01:56:06 STEP 344 H= 344:00 +CPU= 32.973
01:57:00 STEP 345 H= 345:00 +CPU= 51.135
01:57:32 STEP 346 H= 346:00 +CPU= 31.941
01:58:05 STEP 347 H= 347:00 +CPU= 32.714
01:59:02 STEP 348 H= 348:00 +CPU= 54.893
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhq0f+000348
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhq0f+000348
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhq0f+000348
01:59:35 STEP 349 H= 349:00 +CPU= 32.787
02:00:09 STEP 350 H= 350:00 +CPU= 34.581
02:01:01 STEP 351 H= 351:00 +CPU= 51.103
Quit request received from BOINC client, ending the child process
..Failed, model did not complete successfully
(argv0) ../../projects/climateprediction.net/oifs_43r3_ps_1.05_x86_64-pc-linux-gnu
(argv1) start_date: 1982050100
(argv2) exptid: hq0f
(argv3) unique_member_id: 0977
(argv4) batchid: 951
(argv5) wuid: 12168621
(argv6) fclen: 123
(argv7) app_name: oifs_43r3_ps
(argv8) nthreads: 1
Working directory is: /var/lib/boinc-client/slots/3
Project directory is: /var/lib/boinc-client/projects/climateprediction.net/
app name: oifs_43r3_ps
version: 1.05
Location of temp folder: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621
..mkdir for temp folder for results failed
Copying: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip to: /var/lib/boinc-client/slots/3/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc-client/slots/3/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_baad597788e0abfd41ee1eaf1ab4f9da to: /var/lib/boinc-client/slots/3/oifs_43r3_ps_0977_1982050100_123_951_12168621.zip
Unzipping the namelist zip file: /var/lib/boinc-client/slots/3/oifs_43r3_ps_0977_1982050100_123_951_12168621.zip
ic_ancil_file: ic_ancil_12168621
ifsdata_file: ifsdata_12168621
climate_data_file: clim_data_12168621
horiz_resolution: 159
vert_resolution: 60
grid_type: l_2
upload_interval: 24
utstep: 3600
nfrres: restart dump frequency (steps) 24
Copying IC ancils from: ../../projects/climateprediction.net/jf_eb65d99f44920123031e01150753b734 to: /var/lib/boinc-client/slots/3/ic_ancil_12168621.zip
Unzipping the IC ancils zip file: /var/lib/boinc-client/slots/3/ic_ancil_12168621.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_91ea09656a4143e55437ce21e7741399 to: /var/lib/boinc-client/slots/3/ifsdata/ifsdata_12168621.zip
Unzipping the ifsdata_zip file: /var/lib/boinc-client/slots/3/ifsdata/ifsdata_12168621.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc-client/slots/3/159l_2/clim_data_12168621.zip
Unzipping the climate data zip file: /var/lib/boinc-client/slots/3/159l_2/clim_data_12168621.zip
Checking for progress XML file: /var/lib/boinc-client/slots/3/progress_file_12168621.xml
Opened progress file ok : /var/lib/boinc-client/slots/3/progress_file_12168621.xml
-- Model is restarting --
Adjusting last_iter, 352, to previous model restart step.
Creating progress file: /var/lib/boinc-client/slots/3/progress_file_12168621.xml
last_cpu_time: 15205
upload_file_number: 14
last_iter: 335
last_upload: 1209600
model_completed: 0
total_length_of_simulation: 10627200
result_base_name: oifs_43r3_ps_0977_1982050100_123_951_12168621_0_r1802458006
The child process has been launched with process id: 975990
Executing the command: /var/lib/boinc-client/slots/3/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2091] fp = 0x3129fe0
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at (nil))
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:975990:975990] [20221223:020227:1671757347:0.001] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
02:03:48 STEP 336 H= 336:00 +CPU= 57.745
02:04:21 STEP 337 H= 337:00 +CPU= 31.918
02:04:49 STEP 338 H= 338:00 +CPU= 27.889
02:05:40 STEP 339 H= 339:00 +CPU= 51.469
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
02:06:26 STEP 340 H= 340:00 +CPU= 33.930
02:06:58 STEP 341 H= 341:00 +CPU= 31.987
02:07:50 STEP 342 H= 342:00 +CPU= 52.300
02:08:21 STEP 343 H= 343:00 +CPU= 30.427
02:08:55 STEP 344 H= 344:00 +CPU= 34.581
02:09:46 STEP 345 H= 345:00 +CPU= 50.861
02:10:18 STEP 346 H= 346:00 +CPU= 31.203
02:10:49 STEP 347 H= 347:00 +CPU= 31.540
02:11:51 STEP 348 H= 348:00 +CPU= 54.630
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhq0f+000348
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhq0f+000348
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhq0f+000348
02:12:18 STEP 349 H= 349:00 +CPU= 27.139
02:12:51 STEP 350 H= 350:00 +CPU= 32.015
02:13:38 STEP 351 H= 351:00 +CPU= 46.600
02:14:10 STEP 352 H= 352:00 +CPU= 32.761
02:14:41 STEP 353 H= 353:00 +CPU= 30.796
02:15:35 STEP 354 H= 354:00 +CPU= 53.388
02:16:07 STEP 355 H= 355:00 +CPU= 32.036
02:16:40 STEP 356 H= 356:00 +CPU= 33.086
02:17:32 STEP 357 H= 357:00 +CPU= 51.371
02:18:06 STEP 358 H= 358:00 +CPU= 33.894
02:18:39 STEP 359 H= 359:00 +CPU= 31.803
02:19:40 STEP 360 H= 360:00 +CPU= 53.361
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhq0f+000360
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhq0f+000360
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhq0f+000360
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMGGhq0f+000336
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMSHhq0f+000336
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMUAhq0f+000336
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMGGhq0f+000348
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMSHhq0f+000348
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMUAhq0f+000348
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0977_1982050100_123_951_12168621_0_r1802458006_14.zip
Uploading the intermediate file: upload_file_14.zip
02:20:04 STEP 361 H= 361:00 +CPU= 23.378
02:20:37 STEP 362 H= 362:00 +CPU= 32.874
02:21:29 STEP 363 H= 363:00 +CPU= 51.558
02:22:01 STEP 364 H= 364:00 +CPU= 32.081
02:22:34 STEP 365 H= 365:00 +CPU= 33.291
02:23:27 STEP 366 H= 366:00 +CPU= 52.799
02:24:00 STEP 367 H= 367:00 +CPU= 32.849
02:24:32 STEP 368 H= 368:00 +CPU= 31.935
02:25:24 STEP 369 H= 369:00 +CPU= 51.021
02:25:57 STEP 370 H= 370:00 +CPU= 33.283
02:26:30 STEP 371 H= 371:00 +CPU= 32.821
02:27:28 STEP 372 H= 372:00 +CPU= 57.342
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhq0f+000372
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhq0f+000372
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhq0f+000372
02:28:01 STEP 373 H= 373:00 +CPU= 32.673
02:28:35 STEP 374 H= 374:00 +CPU= 33.682
02:29:27 STEP 375 H= 375:00 +CPU= 50.976
02:30:01 STEP 376 H= 376:00 +CPU= 34.130
02:30:32 STEP 377 H= 377:00 +CPU= 31.059
02:31:26 STEP 378 H= 378:00 +CPU= 53.474
02:31:57 STEP 379 H= 379:00 +CPU= 30.943
02:32:28 STEP 380 H= 380:00 +CPU= 31.133
02:33:19 STEP 381 H= 381:00 +CPU= 50.615
02:33:53 STEP 382 H= 382:00 +CPU= 33.707
02:34:26 STEP 383 H= 383:00 +CPU= 31.362
02:35:26 STEP 384 H= 384:00 +CPU= 57.186
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhq0f+000384
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhq0f+000384
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhq0f+000384
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMGGhq0f+000360
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMSHhq0f+000360
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMUAhq0f+000360
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMGGhq0f+000372
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMSHhq0f+000372
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMUAhq0f+000372
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0977_1982050100_123_951_12168621_0_r1802458006_15.zip
Uploading the intermediate file: upload_file_15.zip
02:35:57 STEP 385 H= 385:00 +CPU= 30.959
02:36:31 STEP 386 H= 386:00 +CPU= 34.308
02:37:22 STEP 387 H= 387:00 +CPU= 49.936
02:37:55 STEP 388 H= 388:00 +CPU= 33.701
02:38:26 STEP 389 H= 389:00 +CPU= 30.964
02:39:19 STEP 390 H= 390:00 +CPU= 52.653
02:39:51 STEP 391 H= 391:00 +CPU= 31.636
02:40:24 STEP 392 H= 392:00 +CPU= 32.235
02:41:14 STEP 393 H= 393:00 +CPU= 49.848
02:41:45 STEP 394 H= 394:00 +CPU= 31.381
02:42:16 STEP 395 H= 395:00 +CPU= 30.985
02:43:14 STEP 396 H= 396:00 +CPU= 55.080
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhq0f+000396
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhq0f+000396
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhq0f+000396
02:43:50 STEP 397 H= 397:00 +CPU= 35.037
02:44:23 STEP 398 H= 398:00 +CPU= 32.733
02:45:13 STEP 399 H= 399:00 +CPU= 49.636
02:45:47 STEP 400 H= 400:00 +CPU= 33.978
02:46:19 STEP 401 H= 401:00 +CPU= 31.891
02:47:11 STEP 402 H= 402:00 +CPU= 51.003
02:47:44 STEP 403 H= 403:00 +CPU= 32.983
02:48:16 STEP 404 H= 404:00 +CPU= 32.470
02:49:07 STEP 405 H= 405:00 +CPU= 50.148
02:49:40 STEP 406 H= 406:00 +CPU= 32.983
02:50:13 STEP 407 H= 407:00 +CPU= 32.833
02:51:21 STEP 408 H= 408:00 +CPU= 60.161
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhq0f+000408
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhq0f+000408
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhq0f+000408
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMGGhq0f+000384
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMSHhq0f+000384
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMUAhq0f+000384
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMGGhq0f+000396
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMSHhq0f+000396
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621/ICMUAhq0f+000396
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0977_1982050100_123_951_12168621_0_r1802458006_16.zip
Uploading the intermediate file: upload_file_16.zip
02:51:51 STEP 409 H= 409:00 +CPU= 30.042
02:52:24 STEP 410 H= 410:00 +CPU= 33.218
02:53:14 STEP 411 H= 411:00 +CPU= 49.464
02:53:49 STEP 412 H= 412:00 +CPU= 34.372
02:54:19 STEP 413 H= 413:00 +CPU= 30.422
02:55:11 STEP 414 H= 414:00 +CPU= 51.768
02:55:43 STEP 415 H= 415:00 +CPU= 31.535
02:56:16 STEP 416 H= 416:00 +CPU= 32.235
02:57:09 STEP 417 H= 417:00 +CPU= 52.548
02:57:42 STEP 418 H= 418:00 +CPU= 32.095
02:58:14 STEP 419 H= 419:00 +CPU= 32.009
02:59:13 STEP 420 H= 420:00 +CPU= 56.384
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhq0f+000420
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhq0f+000420
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhq0f+000420
02:59:43 STEP 421 H= 421:00 +CPU= 27.284
03:00:16 STEP 422 H= 422:00 +CPU= 33.328
03:01:08 STEP 423 H= 423:00 +CPU= 51.603
03:01:42 STEP 424 H= 424:00 +CPU= 33.852
03:02:15 STEP 425 H= 425:00 +CPU= 32.437
03:03:08 STEP 426 H= 426:00 +CPU= 53.461
03:03:41 STEP 427 H= 427:00 +CPU= 32.557
03:04:14 STEP 428 H= 428:00 +CPU= 32.547
03:05:07 STEP 429 H= 429:00 +CPU= 53.105
03:05:40 STEP 430 H= 430:00 +CPU= 32.966
03:06:16 STEP 431 H= 431:00 +CPU= 34.498
Quit request received from BOINC client, ending the child process
..Failed, model did not complete successfully
(argv0) ../../projects/climateprediction.net/oifs_43r3_ps_1.05_x86_64-pc-linux-gnu
(argv1) start_date: 1982050100
(argv2) exptid: hq0f
(argv3) unique_member_id: 0977
(argv4) batchid: 951
(argv5) wuid: 12168621
(argv6) fclen: 123
(argv7) app_name: oifs_43r3_ps
(argv8) nthreads: 1
Working directory is: /var/lib/boinc-client/slots/3
Project directory is: /var/lib/boinc-client/projects/climateprediction.net/
app name: oifs_43r3_ps
version: 1.05
Location of temp folder: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12168621
..mkdir for temp folder for results failed
Copying: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip to: /var/lib/boinc-client/slots/3/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc-client/slots/3/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_baad597788e0abfd41ee1eaf1ab4f9da to: /var/lib/boinc-client/slots/3/oifs_43r3_ps_0977_1982050100_123_951_12168621.zip
Unzipping the namelist zip file: /var/lib/boinc-client/slots/3/oifs_43r3_ps_0977_1982050100_123_951_12168621.zip
ic_ancil_file: ic_ancil_12168621
ifsdata_file: ifsdata_12168621
climate_data_file: clim_data_12168621
horiz_resolution: 159
vert_resolution: 60
grid_type: l_2
upload_interval: 24
utstep: 3600
nfrres: restart dump frequency (steps) 24
Copying IC ancils from: ../../projects/climateprediction.net/jf_eb65d99f44920123031e01150753b734 to: /var/lib/boinc-client/slots/3/ic_ancil_12168621.zip
Unzipping the IC ancils zip file: /var/lib/boinc-client/slots/3/ic_ancil_12168621.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_91ea09656a4143e55437ce21e7741399 to: /var/lib/boinc-client/slots/3/ifsdata/ifsdata_12168621.zip
Unzipping the ifsdata_zip file: /var/lib/boinc-client/slots/3/ifsdata/ifsdata_12168621.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc-client/slots/3/159l_2/clim_data_12168621.zip
Unzipping the climate data zip file: /var/lib/boinc-client/slots/3/159l_2/clim_data_12168621.zip
Checking for progress XML file: /var/lib/boinc-client/slots/3/progress_file_12168621.xml
Opened progress file ok : /var/lib/boinc-client/slots/3/progress_file_12168621.xml
-- Model is restarting --
Adjusting last_iter, 431, to previous model restart step.
Creating progress file: /var/lib/boinc-client/slots/3/progress_file_12168621.xml
last_cpu_time: 18942
upload_file_number: 17
last_iter: 407
last_upload: 1468800
model_completed: 0
total_length_of_simulation: 10627200
result_base_name: oifs_43r3_ps_0977_1982050100_123_951_12168621_0_r1802458006
The child process has been launched with process id: 980438
Executing the command: /var/lib/boinc-client/slots/3/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2091] fp = 0x3129fe0
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at (nil))
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:030751:1671761271:0.000] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
ABORT! 1 !! *** WAVE MODEL HAS ABORTED
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:031000:1671761400:128.769] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3843] 1721 MB (maxheap), 3520 MB (maxrss), 0 MB (maxstack)
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:031000:1671761400:128.769] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : MASTER
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:031000:1671761400:128.769] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT0
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:031000:1671761400:128.769] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:031000:1671761400:128.769] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT2
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:031000:1671761400:128.769] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT3
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:031000:1671761400:128.769] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT4
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:031000:1671761400:128.769] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : WVCOUPLE
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:031000:1671761400:128.769] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : WAVEMDL
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:031000:1671761400:128.769] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : INITMDL
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:031000:1671761400:128.769] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : USERIN
SDL_TRACEBACK: Calling INTEL_TRBK, THRD = 1
Process 1 thread 1 calling linux_trbk from intel_trbk()
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:240] Backtrace(s) for program '/var/lib/boinc-client/slots/3/oifs_43r3_model.exe' : sigcontextptr=0x7ffd5034b220
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:277] Backtrace (size = 19) with addr2line-cmd
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:302] /usr/bin/addr2line -fs -e '/var/lib/boinc-client/slots/3/oifs_43r3_model.exe' 0x839273 0x839879 0x8c2957 0x81d89e 0x15f8aca 0x15d51c8 0x166b07f 0x15d7ff0 0x8711e0 0x7e0165 0x55e4ad 0x5471c9 0x546412 0x545e28 0x40708d 0x40220f 0x4021a2 0x1dca390 0x40206e
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [0]: [0x839273] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [1]: [0x839879] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [2]: [0x8c2957] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [3]: [0x81d89e] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [4]: [0x15f8aca] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [5]: [0x15d51c8] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [6]: [0x166b07f] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [7]: [0x15d7ff0] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [8]: [0x8711e0] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [9]: [0x7e0165] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [10]: [0x55e4ad] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [11]: [0x5471c9] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [12]: [0x546412] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [13]: [0x545e28] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [14]: [0x40708d] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [15]: [0x40220f] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [16]: [0x4021a2] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [17]: [0x1dca390] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [18]: [0x40206e] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:370] End of backtrace(s)
SDL_TRACEBACK: Done INTEL_TRBK, THRD = 1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:031005:1671761405:133.935] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1538] Received signal#6 (SIGABRT) :: 1721MB (heap), 3520MB (maxrss), 0MB (maxstack), 0 (paging), nsigs = 1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:031005:1671761405:133.935] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1542] Also activating Harakiri-alarm (SIGALRM=14) to expire after 500s elapsed to prevent hangs, nsigs = 1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:031005:1671761405:133.935] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1544] Harakiri signal handler 'signal_harakiri' for signal#14 (SIGALRM) installed at 0x81f3c0 (old at (nil))
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:031005:1671761405:133.935] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1617] Signal#6 was caused by unrecognized si_code [memaddr=0x7c000ef5d6], nsigs = 1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:031005:1671761405:133.935] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1686] Starting DrHook backtrace for signal#6, nsigs = 1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:031006:1671761406:134.935] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1734] DrHook backtrace done for signal#6, nsigs = 1
[EC_DRHOOK:thomas-HP-ENVY-dv7-Notebook-PC:1:1:980438:980438] [20221223:031006:1671761406:134.935] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1785] Calling previous signal handler at 0x1ce8bb0 for signal#6, nsigs = 1
forrtl: error (76): Abort trap signal
Image PC Routine Line Source
oifs_43r3_model.e 0000000001CE916B Unknown Unknown Unknown
oifs_43r3_model.e 000000000081FFF1 Unknown Unknown Unknown
oifs_43r3_model.e 0000000001DC9090 Unknown Unknown Unknown
oifs_43r3_model.e 0000000001DC903B Unknown Unknown Unknown
oifs_43r3_model.e 00000000008C2880 sdl_mod_mp_sdl_sr 100 sdl_mod.F90
oifs_43r3_model.e 000000000081D8AF abor1_ 41 abor1.F90
oifs_43r3_model.e 00000000015F8ACA mpabort_ 50 mpabort.F
oifs_43r3_model.e 00000000015D51C8 abort1_ 44 abort1.F
oifs_43r3_model.e 000000000166B07F userin_ 553 userin.F
oifs_43r3_model.e 00000000015D7FF0 initmdl_ 452 initmdl.F
oifs_43r3_model.e 00000000008711E0 wavemdl_ 393 wavemdl.F
oifs_43r3_model.e 00000000007E0165 wvcouple_ 260 wvcouple.F90
oifs_43r3_model.e 000000000055E4AD cnt4_ 1283 cnt4.F90
oifs_43r3_model.e 00000000005471C9 cnt3_ 267 cnt3.F90
oifs_43r3_model.e 0000000000546412 cnt2_ 88 cnt2.F90
oifs_43r3_model.e 0000000000545E28 cnt1_ 92 cnt1.F90
oifs_43r3_model.e 000000000040708D cnt0_ 146 cnt0.F90
oifs_43r3_model.e 000000000040220F MAIN__ 96 master.F90
oifs_43r3_model.e 00000000004021A2 Unknown Unknown Unknown
oifs_43r3_model.e 0000000001DCA390 Unknown Unknown Unknown
oifs_43r3_model.e 000000000040206E Unknown Unknown Unknown
The child process terminated with status: 0
..Failed, model did not complete successfully
</stderr_txt>
]]>
