<core_client_version>7.16.16</core_client_version>
<![CDATA[
<message>
process exited with code 1 (0x1, -255)</message>
<stderr_txt>
290:00 +CPU= 51.179
18:50:37 STEP 291 H= 291:00 +CPU= 80.275
18:51:36 STEP 292 H= 292:00 +CPU= 57.631
18:52:27 STEP 293 H= 293:00 +CPU= 50.079
18:53:51 STEP 294 H= 294:00 +CPU= 82.565
18:54:46 STEP 295 H= 295:00 +CPU= 54.365
18:55:41 STEP 296 H= 296:00 +CPU= 52.966
18:56:52 STEP 297 H= 297:00 +CPU= 55.659
18:57:30 STEP 298 H= 298:00 +CPU= 26.572
18:57:58 STEP 299 H= 299:00 +CPU= 26.179
18:58:43 STEP 300 H= 300:00 +CPU= 43.075
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMGGhq0f+000300
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMSHhq0f+000300
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMUAhq0f+000300
18:59:09 STEP 301 H= 301:00 +CPU= 25.444
18:59:41 STEP 302 H= 302:00 +CPU= 31.923
19:00:44 STEP 303 H= 303:00 +CPU= 41.028
19:01:23 STEP 304 H= 304:00 +CPU= 22.508
19:01:47 STEP 305 H= 305:00 +CPU= 23.347
19:02:23 STEP 306 H= 306:00 +CPU= 36.011
19:02:50 STEP 307 H= 307:00 +CPU= 26.038
19:03:21 STEP 308 H= 308:00 +CPU= 30.944
19:04:06 STEP 309 H= 309:00 +CPU= 44.113
19:04:41 STEP 310 H= 310:00 +CPU= 33.330
19:05:28 STEP 311 H= 311:00 +CPU= 36.417
19:07:46 STEP 312 H= 312:00 +CPU= 70.731
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMGGhq0f+000312
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMSHhq0f+000312
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMUAhq0f+000312
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMGGhq0f+000288
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMSHhq0f+000288
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMUAhq0f+000288
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMGGhq0f+000300
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMSHhq0f+000300
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMUAhq0f+000300
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0009_1992050100_123_961_12177653_1_r721985956_12.zip
Uploading the intermediate file: upload_file_12.zip
19:08:27 STEP 313 H= 313:00 +CPU= 40.218
19:09:06 STEP 314 H= 314:00 +CPU= 36.093
19:09:59 STEP 315 H= 315:00 +CPU= 52.185
19:10:35 STEP 316 H= 316:00 +CPU= 35.676
19:11:10 STEP 317 H= 317:00 +CPU= 34.709
19:12:11 STEP 318 H= 318:00 +CPU= 54.494
19:12:43 STEP 319 H= 319:00 +CPU= 32.501
19:13:29 STEP 320 H= 320:00 +CPU= 38.548
19:14:23 STEP 321 H= 321:00 +CPU= 52.442
19:14:54 STEP 322 H= 322:00 +CPU= 31.059
19:15:26 STEP 323 H= 323:00 +CPU= 31.180
19:16:27 STEP 324 H= 324:00 +CPU= 58.879
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMGGhq0f+000324
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMSHhq0f+000324
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMUAhq0f+000324
19:17:02 STEP 325 H= 325:00 +CPU= 34.418
19:17:37 STEP 326 H= 326:00 +CPU= 34.836
19:18:39 STEP 327 H= 327:00 +CPU= 61.575
19:19:25 STEP 328 H= 328:00 +CPU= 45.344
19:20:10 STEP 329 H= 329:00 +CPU= 43.530
19:21:19 STEP 330 H= 330:00 +CPU= 67.946
19:22:04 STEP 331 H= 331:00 +CPU= 44.338
19:23:27 STEP 332 H= 332:00 +CPU= 30.062
19:24:06 STEP 333 H= 333:00 +CPU= 35.735
19:24:31 STEP 334 H= 334:00 +CPU= 24.166
19:25:02 STEP 335 H= 335:00 +CPU= 29.292
19:27:27 STEP 336 H= 336:00 +CPU= 45.266
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMGGhq0f+000336
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMSHhq0f+000336
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMUAhq0f+000336
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMGGhq0f+000312
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMSHhq0f+000312
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMUAhq0f+000312
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMGGhq0f+000324
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMSHhq0f+000324
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMUAhq0f+000324
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0009_1992050100_123_961_12177653_1_r721985956_13.zip
Uploading the intermediate file: upload_file_13.zip
19:27:50 STEP 337 H= 337:00 +CPU= 23.171
19:28:15 STEP 338 H= 338:00 +CPU= 24.046
19:28:59 STEP 339 H= 339:00 +CPU= 43.117
19:29:31 STEP 340 H= 340:00 +CPU= 32.344
19:30:08 STEP 341 H= 341:00 +CPU= 35.837
19:30:59 STEP 342 H= 342:00 +CPU= 50.338
19:31:34 STEP 343 H= 343:00 +CPU= 34.687
19:32:13 STEP 344 H= 344:00 +CPU= 38.942
19:33:13 STEP 345 H= 345:00 +CPU= 59.281
19:33:55 STEP 346 H= 346:00 +CPU= 41.116
19:34:36 STEP 347 H= 347:00 +CPU= 40.462
19:35:43 STEP 348 H= 348:00 +CPU= 66.782
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMGGhq0f+000348
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMSHhq0f+000348
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMUAhq0f+000348
19:36:23 STEP 349 H= 349:00 +CPU= 39.282
19:37:01 STEP 350 H= 350:00 +CPU= 37.338
19:38:35 STEP 351 H= 351:00 +CPU= 47.595
19:39:04 STEP 352 H= 352:00 +CPU= 23.420
19:39:31 STEP 353 H= 353:00 +CPU= 26.488
19:40:21 STEP 354 H= 354:00 +CPU= 48.723
19:40:59 STEP 355 H= 355:00 +CPU= 35.012
19:41:41 STEP 356 H= 356:00 +CPU= 37.735
19:42:47 STEP 357 H= 357:00 +CPU= 55.621
19:43:25 STEP 358 H= 358:00 +CPU= 37.331
19:44:13 STEP 359 H= 359:00 +CPU= 46.719
19:46:31 STEP 360 H= 360:00 +CPU= 72.200
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMGGhq0f+000360
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMSHhq0f+000360
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMUAhq0f+000360
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMGGhq0f+000336
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMSHhq0f+000336
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMUAhq0f+000336
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMGGhq0f+000348
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMSHhq0f+000348
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMUAhq0f+000348
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0009_1992050100_123_961_12177653_1_r721985956_14.zip
Uploading the intermediate file: upload_file_14.zip
19:47:10 STEP 361 H= 361:00 +CPU= 36.873
19:47:44 STEP 362 H= 362:00 +CPU= 34.422
19:48:30 STEP 363 H= 363:00 +CPU= 44.490
19:49:09 STEP 364 H= 364:00 +CPU= 38.660
19:49:47 STEP 365 H= 365:00 +CPU= 35.491
19:50:41 STEP 366 H= 366:00 +CPU= 53.402
19:51:25 STEP 367 H= 367:00 +CPU= 43.681
19:52:15 STEP 368 H= 368:00 +CPU= 48.852
19:53:28 STEP 369 H= 369:00 +CPU= 72.540
19:54:23 STEP 370 H= 370:00 +CPU= 53.972
19:55:17 STEP 371 H= 371:00 +CPU= 53.137
19:56:49 STEP 372 H= 372:00 +CPU= 90.795
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMGGhq0f+000372
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMSHhq0f+000372
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMUAhq0f+000372
19:57:43 STEP 373 H= 373:00 +CPU= 52.842
19:58:36 STEP 374 H= 374:00 +CPU= 51.664
19:59:51 STEP 375 H= 375:00 +CPU= 73.530
20:00:40 STEP 376 H= 376:00 +CPU= 48.408
20:01:28 STEP 377 H= 377:00 +CPU= 47.831
20:02:38 STEP 378 H= 378:00 +CPU= 68.724
20:03:26 STEP 379 H= 379:00 +CPU= 46.952
20:04:20 STEP 380 H= 380:00 +CPU= 53.031
20:05:43 STEP 381 H= 381:00 +CPU= 81.980
20:06:39 STEP 382 H= 382:00 +CPU= 55.178
20:07:36 STEP 383 H= 383:00 +CPU= 56.255
20:09:11 STEP 384 H= 384:00 +CPU= 89.901
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMGGhq0f+000384
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMSHhq0f+000384
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMUAhq0f+000384
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMGGhq0f+000360
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMSHhq0f+000360
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMUAhq0f+000360
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMGGhq0f+000372
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMSHhq0f+000372
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMUAhq0f+000372
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0009_1992050100_123_961_12177653_1_r721985956_15.zip
Uploading the intermediate file: upload_file_15.zip
20:10:03 STEP 385 H= 385:00 +CPU= 51.616
20:11:02 STEP 386 H= 386:00 +CPU= 51.043
20:12:13 STEP 387 H= 387:00 +CPU= 68.762
20:12:59 STEP 388 H= 388:00 +CPU= 45.770
20:13:43 STEP 389 H= 389:00 +CPU= 43.334
20:14:54 STEP 390 H= 390:00 +CPU= 70.093
20:15:43 STEP 391 H= 391:00 +CPU= 48.099
20:16:34 STEP 392 H= 392:00 +CPU= 49.752
20:17:59 STEP 393 H= 393:00 +CPU= 83.845
20:18:56 STEP 394 H= 394:00 +CPU= 56.448
20:19:51 STEP 395 H= 395:00 +CPU= 53.594
20:21:25 STEP 396 H= 396:00 +CPU= 92.387
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMGGhq0f+000396
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMSHhq0f+000396
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMUAhq0f+000396
20:22:21 STEP 397 H= 397:00 +CPU= 55.238
20:23:18 STEP 398 H= 398:00 +CPU= 56.144
20:24:36 STEP 399 H= 399:00 +CPU= 76.116
20:25:29 STEP 400 H= 400:00 +CPU= 52.100
20:26:18 STEP 401 H= 401:00 +CPU= 47.984
20:27:28 STEP 402 H= 402:00 +CPU= 69.200
20:28:16 STEP 403 H= 403:00 +CPU= 46.840
20:29:05 STEP 404 H= 404:00 +CPU= 46.553
20:30:23 STEP 405 H= 405:00 +CPU= 77.475
20:31:17 STEP 406 H= 406:00 +CPU= 52.295
20:32:10 STEP 407 H= 407:00 +CPU= 52.002
20:33:42 STEP 408 H= 408:00 +CPU= 89.360
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMGGhq0f+000408
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMSHhq0f+000408
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMUAhq0f+000408
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMGGhq0f+000384
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMSHhq0f+000384
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMUAhq0f+000384
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMGGhq0f+000396
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMSHhq0f+000396
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMUAhq0f+000396
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0009_1992050100_123_961_12177653_1_r721985956_16.zip
Uploading the intermediate file: upload_file_16.zip
20:34:35 STEP 409 H= 409:00 +CPU= 51.570
20:35:27 STEP 410 H= 410:00 +CPU= 51.540
20:36:35 STEP 411 H= 411:00 +CPU= 66.952
20:37:23 STEP 412 H= 412:00 +CPU= 46.542
20:38:09 STEP 413 H= 413:00 +CPU= 45.221
20:39:20 STEP 414 H= 414:00 +CPU= 68.071
20:40:06 STEP 415 H= 415:00 +CPU= 45.599
20:40:51 STEP 416 H= 416:00 +CPU= 42.481
20:42:00 STEP 417 H= 417:00 +CPU= 68.343
20:42:43 STEP 418 H= 418:00 +CPU= 42.161
20:43:25 STEP 419 H= 419:00 +CPU= 42.153
20:45:50 STEP 420 H= 420:00 +CPU= 72.668
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMGGhq0f+000420
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMSHhq0f+000420
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMUAhq0f+000420
20:46:24 STEP 421 H= 421:00 +CPU= 32.678
20:50:15 STEP 422 H= 422:00 +CPU= 46.396
Quit request received from BOINC client, ending the child process
(argv0) ../../projects/climateprediction.net/oifs_43r3_ps_1.05_x86_64-pc-linux-gnu
(argv1) start_date: 1992050100
(argv2) exptid: hq0f
(argv3) unique_member_id: 0009
(argv4) batchid: 961
(argv5) wuid: 12177653
(argv6) fclen: 123
(argv7) app_name: oifs_43r3_ps
(argv8) nthreads: 1
Working directory is: /var/lib/boinc-client/slots/9
Project directory is: /var/lib/boinc-client/projects/climateprediction.net/
app name: oifs_43r3_ps
version: 1.05
Location of temp folder: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653
..mkdir for temp folder for results failed
Copying: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip to: /var/lib/boinc-client/slots/9/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc-client/slots/9/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_a1fcf8a5b64521898051a80058ab32b3 to: /var/lib/boinc-client/slots/9/oifs_43r3_ps_0009_1992050100_123_961_12177653.zip
Unzipping the namelist zip file: /var/lib/boinc-client/slots/9/oifs_43r3_ps_0009_1992050100_123_961_12177653.zip
ic_ancil_file: ic_ancil_12177653
ifsdata_file: ifsdata_12177653
climate_data_file: clim_data_12177653
horiz_resolution: 159
vert_resolution: 60
grid_type: l_2
upload_interval: 24
utstep: 3600
nfrres: restart dump frequency (steps) 24
Copying IC ancils from: ../../projects/climateprediction.net/jf_9a5d01b7017216030a390d6462d7d2f6 to: /var/lib/boinc-client/slots/9/ic_ancil_12177653.zip
Unzipping the IC ancils zip file: /var/lib/boinc-client/slots/9/ic_ancil_12177653.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_1a1280b7d69ea1eaef8e189a2a2f7d62 to: /var/lib/boinc-client/slots/9/ifsdata/ifsdata_12177653.zip
Unzipping the ifsdata_zip file: /var/lib/boinc-client/slots/9/ifsdata/ifsdata_12177653.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc-client/slots/9/159l_2/clim_data_12177653.zip
Unzipping the climate data zip file: /var/lib/boinc-client/slots/9/159l_2/clim_data_12177653.zip
Checking for progress XML file: /var/lib/boinc-client/slots/9/progress_file_12177653.xml
Opened progress file ok : /var/lib/boinc-client/slots/9/progress_file_12177653.xml
-- Model is restarting --
Adjusting last_iter, 423, to previous model restart step.
Creating progress file: /var/lib/boinc-client/slots/9/progress_file_12177653.xml
last_cpu_time: 20496
upload_file_number: 17
last_iter: 407
last_upload: 1468800
model_completed: 0
total_length_of_simulation: 10627200
result_base_name: oifs_43r3_ps_0009_1992050100_123_961_12177653_1_r721985956
The child process has been launched with process id: 624451
Executing the command: /var/lib/boinc-client/slots/9/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2091] fp = 0x3129fe0
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.001] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.001] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.001] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at (nil))
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:Debian12:1:1:624451:624451] [20230118:205840:1674075520:0.001] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
21:02:17 STEP 408 H= 408:00 +CPU= 82.833
21:03:07 STEP 409 H= 409:00 +CPU= 44.549
21:03:53 STEP 410 H= 410:00 +CPU= 45.741
21:05:14 STEP 411 H= 411:00 +CPU= 71.946
21:06:04 STEP 412 H= 412:00 +CPU= 49.483
21:06:50 STEP 413 H= 413:00 +CPU= 45.372
21:08:04 STEP 414 H= 414:00 +CPU= 72.266
21:08:54 STEP 415 H= 415:00 +CPU= 44.548
21:09:27 STEP 416 H= 416:00 +CPU= 32.462
21:10:16 STEP 417 H= 417:00 +CPU= 38.298
21:10:51 STEP 418 H= 418:00 +CPU= 34.998
21:11:36 STEP 419 H= 419:00 +CPU= 41.677
21:12:44 STEP 420 H= 420:00 +CPU= 55.682
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMGGhq0f+000420
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMSHhq0f+000420
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMUAhq0f+000420
21:13:11 STEP 421 H= 421:00 +CPU= 26.292
21:13:40 STEP 422 H= 422:00 +CPU= 29.172
21:14:26 STEP 423 H= 423:00 +CPU= 45.552
21:15:00 STEP 424 H= 424:00 +CPU= 33.015
21:15:36 STEP 425 H= 425:00 +CPU= 34.437
21:16:30 STEP 426 H= 426:00 +CPU= 53.567
21:17:01 STEP 427 H= 427:00 +CPU= 30.554
21:17:37 STEP 428 H= 428:00 +CPU= 35.708
21:18:35 STEP 429 H= 429:00 +CPU= 58.292
21:19:19 STEP 430 H= 430:00 +CPU= 37.895
21:19:53 STEP 431 H= 431:00 +CPU= 32.612
21:21:25 STEP 432 H= 432:00 +CPU= 77.543
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMGGhq0f+000432
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMSHhq0f+000432
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMUAhq0f+000432
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMGGhq0f+000408
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMSHhq0f+000408
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMUAhq0f+000408
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMGGhq0f+000420
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMSHhq0f+000420
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMUAhq0f+000420
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0009_1992050100_123_961_12177653_1_r721985956_17.zip
Uploading the intermediate file: upload_file_17.zip
21:22:23 STEP 433 H= 433:00 +CPU= 44.267
21:22:54 STEP 434 H= 434:00 +CPU= 30.129
21:24:02 STEP 435 H= 435:00 +CPU= 67.247
21:25:43 STEP 436 H= 436:00 +CPU= 46.665
21:26:28 STEP 437 H= 437:00 +CPU= 34.831
21:28:00 STEP 438 H= 438:00 +CPU= 73.971
21:28:48 STEP 439 H= 439:00 +CPU= 44.839
21:30:31 STEP 440 H= 440:00 +CPU= 38.822
21:33:37 STEP 441 H= 441:00 +CPU= 69.214
21:34:49 STEP 442 H= 442:00 +CPU= 43.477
21:35:36 STEP 443 H= 443:00 +CPU= 45.985
21:36:57 STEP 444 H= 444:00 +CPU= 74.563
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMGGhq0f+000444
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMSHhq0f+000444
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMUAhq0f+000444
21:37:45 STEP 445 H= 445:00 +CPU= 47.397
21:38:35 STEP 446 H= 446:00 +CPU= 49.412
21:39:39 STEP 447 H= 447:00 +CPU= 62.109
21:40:21 STEP 448 H= 448:00 +CPU= 41.257
21:41:04 STEP 449 H= 449:00 +CPU= 43.154
21:42:13 STEP 450 H= 450:00 +CPU= 67.699
21:43:02 STEP 451 H= 451:00 +CPU= 48.767
21:43:57 STEP 452 H= 452:00 +CPU= 53.798
21:45:04 STEP 453 H= 453:00 +CPU= 53.741
21:45:46 STEP 454 H= 454:00 +CPU= 41.094
21:46:36 STEP 455 H= 455:00 +CPU= 49.187
21:47:57 STEP 456 H= 456:00 +CPU= 61.113
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMGGhq0f+000456
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMSHhq0f+000456
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMUAhq0f+000456
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMGGhq0f+000432
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMSHhq0f+000432
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMUAhq0f+000432
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMGGhq0f+000444
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMSHhq0f+000444
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMUAhq0f+000444
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0009_1992050100_123_961_12177653_1_r721985956_18.zip
Uploading the intermediate file: upload_file_18.zip
21:48:26 STEP 457 H= 457:00 +CPU= 29.278
21:49:08 STEP 458 H= 458:00 +CPU= 41.512
21:50:33 STEP 459 H= 459:00 +CPU= 70.405
21:52:02 STEP 460 H= 460:00 +CPU= 42.976
21:52:39 STEP 461 H= 461:00 +CPU= 36.956
21:54:46 STEP 462 H= 462:00 +CPU= 73.440
21:57:08 STEP 463 H= 463:00 +CPU= 44.343
21:57:52 STEP 464 H= 464:00 +CPU= 41.257
21:58:58 STEP 465 H= 465:00 +CPU= 62.352
21:59:46 STEP 466 H= 466:00 +CPU= 46.591
22:00:17 STEP 467 H= 467:00 +CPU= 30.798
22:01:29 STEP 468 H= 468:00 +CPU= 60.782
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMGGhq0f+000468
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMSHhq0f+000468
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMUAhq0f+000468
22:02:17 STEP 469 H= 469:00 +CPU= 47.371
22:03:15 STEP 470 H= 470:00 +CPU= 50.367
22:04:05 STEP 471 H= 471:00 +CPU= 44.665
22:04:32 STEP 472 H= 472:00 +CPU= 25.972
22:04:57 STEP 473 H= 473:00 +CPU= 24.886
22:06:46 STEP 474 H= 474:00 +CPU= 55.864
22:07:27 STEP 475 H= 475:00 +CPU= 34.849
22:09:38 STEP 476 H= 476:00 +CPU= 43.037
22:10:44 STEP 477 H= 477:00 +CPU= 61.662
22:11:33 STEP 478 H= 478:00 +CPU= 48.705
22:12:22 STEP 479 H= 479:00 +CPU= 48.004
Quit request received from BOINC client, ending the child process
(argv0) ../../projects/climateprediction.net/oifs_43r3_ps_1.05_x86_64-pc-linux-gnu
(argv1) start_date: 1992050100
(argv2) exptid: hq0f
(argv3) unique_member_id: 0009
(argv4) batchid: 961
(argv5) wuid: 12177653
(argv6) fclen: 123
(argv7) app_name: oifs_43r3_ps
(argv8) nthreads: 1
Working directory is: /var/lib/boinc-client/slots/9
Project directory is: /var/lib/boinc-client/projects/climateprediction.net/
app name: oifs_43r3_ps
version: 1.05
Location of temp folder: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653
..mkdir for temp folder for results failed
Copying: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip to: /var/lib/boinc-client/slots/9/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc-client/slots/9/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_a1fcf8a5b64521898051a80058ab32b3 to: /var/lib/boinc-client/slots/9/oifs_43r3_ps_0009_1992050100_123_961_12177653.zip
Unzipping the namelist zip file: /var/lib/boinc-client/slots/9/oifs_43r3_ps_0009_1992050100_123_961_12177653.zip
ic_ancil_file: ic_ancil_12177653
ifsdata_file: ifsdata_12177653
climate_data_file: clim_data_12177653
horiz_resolution: 159
vert_resolution: 60
grid_type: l_2
upload_interval: 24
utstep: 3600
nfrres: restart dump frequency (steps) 24
Copying IC ancils from: ../../projects/climateprediction.net/jf_9a5d01b7017216030a390d6462d7d2f6 to: /var/lib/boinc-client/slots/9/ic_ancil_12177653.zip
Unzipping the IC ancils zip file: /var/lib/boinc-client/slots/9/ic_ancil_12177653.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_1a1280b7d69ea1eaef8e189a2a2f7d62 to: /var/lib/boinc-client/slots/9/ifsdata/ifsdata_12177653.zip
Unzipping the ifsdata_zip file: /var/lib/boinc-client/slots/9/ifsdata/ifsdata_12177653.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc-client/slots/9/159l_2/clim_data_12177653.zip
Unzipping the climate data zip file: /var/lib/boinc-client/slots/9/159l_2/clim_data_12177653.zip
Checking for progress XML file: /var/lib/boinc-client/slots/9/progress_file_12177653.xml
Opened progress file ok : /var/lib/boinc-client/slots/9/progress_file_12177653.xml
-- Model is restarting --
Adjusting last_iter, 480, to previous model restart step.
Creating progress file: /var/lib/boinc-client/slots/9/progress_file_12177653.xml
last_cpu_time: 23973
upload_file_number: 19
last_iter: 479
last_upload: 1641600
model_completed: 0
total_length_of_simulation: 10627200
result_base_name: oifs_43r3_ps_0009_1992050100_123_961_12177653_1_r721985956
The child process has been launched with process id: 624707
Executing the command: /var/lib/boinc-client/slots/9/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2091] fp = 0x3129fe0
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.000] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.000] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.001] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at (nil))
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:Debian12:1:1:624707:624707] [20230118:221642:1674080202:0.001] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
22:18:17 STEP 480 H= 480:00 +CPU= 83.535
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMGGhq0f+000480
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMSHhq0f+000480
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMUAhq0f+000480
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMGGhq0f+000456
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMSHhq0f+000456
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMUAhq0f+000456
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMGGhq0f+000468
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMSHhq0f+000468
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMUAhq0f+000468
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0009_1992050100_123_961_12177653_1_r721985956_19.zip
Uploading the intermediate file: upload_file_19.zip
22:19:09 STEP 481 H= 481:00 +CPU= 29.074
22:19:40 STEP 482 H= 482:00 +CPU= 29.644
22:20:46 STEP 483 H= 483:00 +CPU= 48.310
22:21:17 STEP 484 H= 484:00 +CPU= 31.106
22:21:51 STEP 485 H= 485:00 +CPU= 33.057
22:22:47 STEP 486 H= 486:00 +CPU= 55.661
22:23:22 STEP 487 H= 487:00 +CPU= 34.489
22:23:55 STEP 488 H= 488:00 +CPU= 32.587
22:24:48 STEP 489 H= 489:00 +CPU= 53.090
22:25:24 STEP 490 H= 490:00 +CPU= 35.649
22:25:57 STEP 491 H= 491:00 +CPU= 32.207
22:26:59 STEP 492 H= 492:00 +CPU= 60.443
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMGGhq0f+000492
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMSHhq0f+000492
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMUAhq0f+000492
22:27:40 STEP 493 H= 493:00 +CPU= 36.976
22:28:27 STEP 494 H= 494:00 +CPU= 42.835
22:29:34 STEP 495 H= 495:00 +CPU= 63.197
22:30:07 STEP 496 H= 496:00 +CPU= 31.936
22:30:46 STEP 497 H= 497:00 +CPU= 38.766
22:31:47 STEP 498 H= 498:00 +CPU= 61.138
22:32:23 STEP 499 H= 499:00 +CPU= 35.174
22:33:04 STEP 500 H= 500:00 +CPU= 40.469
22:34:04 STEP 501 H= 501:00 +CPU= 59.502
22:34:42 STEP 502 H= 502:00 +CPU= 38.003
22:35:27 STEP 503 H= 503:00 +CPU= 43.811
22:36:53 STEP 504 H= 504:00 +CPU= 71.606
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMGGhq0f+000504
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMSHhq0f+000504
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMUAhq0f+000504
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMGGhq0f+000480
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMSHhq0f+000480
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMUAhq0f+000480
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMGGhq0f+000492
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMSHhq0f+000492
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMUAhq0f+000492
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0009_1992050100_123_961_12177653_1_r721985956_20.zip
Uploading the intermediate file: upload_file_20.zip
22:37:16 STEP 505 H= 505:00 +CPU= 22.873
22:37:56 STEP 506 H= 506:00 +CPU= 39.337
22:38:55 STEP 507 H= 507:00 +CPU= 57.793
22:39:30 STEP 508 H= 508:00 +CPU= 34.450
22:40:06 STEP 509 H= 509:00 +CPU= 35.450
22:41:06 STEP 510 H= 510:00 +CPU= 59.217
22:41:34 STEP 511 H= 511:00 +CPU= 27.809
22:42:16 STEP 512 H= 512:00 +CPU= 41.444
22:43:45 STEP 513 H= 513:00 +CPU= 68.830
22:44:24 STEP 514 H= 514:00 +CPU= 37.044
22:45:03 STEP 515 H= 515:00 +CPU= 38.268
22:46:15 STEP 516 H= 516:00 +CPU= 69.684
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMGGhq0f+000516
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMSHhq0f+000516
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMUAhq0f+000516
22:46:50 STEP 517 H= 517:00 +CPU= 34.786
22:47:33 STEP 518 H= 518:00 +CPU= 41.814
22:48:39 STEP 519 H= 519:00 +CPU= 63.155
22:49:37 STEP 520 H= 520:00 +CPU= 31.538
22:50:16 STEP 521 H= 521:00 +CPU= 38.156
22:54:03 STEP 522 H= 522:00 +CPU= 65.731
22:54:57 STEP 523 H= 523:00 +CPU= 37.538
22:55:35 STEP 524 H= 524:00 +CPU= 37.298
22:57:15 STEP 525 H= 525:00 +CPU= 62.227
22:57:51 STEP 526 H= 526:00 +CPU= 35.276
22:58:44 STEP 527 H= 527:00 +CPU= 44.675
23:00:09 STEP 528 H= 528:00 +CPU= 48.424
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMGGhq0f+000528
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMSHhq0f+000528
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMUAhq0f+000528
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMGGhq0f+000504
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMSHhq0f+000504
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMUAhq0f+000504
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMGGhq0f+000516
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMSHhq0f+000516
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653/ICMUAhq0f+000516
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0009_1992050100_123_961_12177653_1_r721985956_21.zip
Uploading the intermediate file: upload_file_21.zip
23:07:52 STEP 529 H= 529:00 +CPU= 46.894
23:09:05 STEP 530 H= 530:00 +CPU= 47.864
23:11:03 STEP 531 H= 531:00 +CPU= 49.937
23:12:57 STEP 532 H= 532:00 +CPU= 39.477
23:13:36 STEP 533 H= 533:00 +CPU= 38.591
23:14:48 STEP 534 H= 534:00 +CPU= 56.171
23:15:27 STEP 535 H= 535:00 +CPU= 38.729
23:16:08 STEP 536 H= 536:00 +CPU= 39.396
23:17:49 STEP 537 H= 537:00 +CPU= 50.350
23:18:09 STEP 538 H= 538:00 +CPU= 19.653
23:18:38 STEP 539 H= 539:00 +CPU= 28.748
23:19:57 STEP 540 H= 540:00 +CPU= 65.573
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMGGhq0f+000540
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMSHhq0f+000540
Moving to projects directory: /var/lib/boinc-client/slots/9/ICMUAhq0f+000540
23:20:37 STEP 541 H= 541:00 +CPU= 39.288
23:21:22 STEP 542 H= 542:00 +CPU= 44.544
23:22:35 STEP 543 H= 543:00 +CPU= 72.011
23:23:25 STEP 544 H= 544:00 +CPU= 48.473
23:24:13 STEP 545 H= 545:00 +CPU= 47.589
23:25:28 STEP 546 H= 546:00 +CPU= 73.216
23:26:15 STEP 547 H= 547:00 +CPU= 46.758
23:27:05 STEP 548 H= 548:00 +CPU= 48.912
23:28:18 STEP 549 H= 549:00 +CPU= 71.466
23:29:06 STEP 550 H= 550:00 +CPU= 47.896
23:29:57 STEP 551 H= 551:00 +CPU= 49.590
Quit request received from BOINC client, ending the child process
(argv0) ../../projects/climateprediction.net/oifs_43r3_ps_1.05_x86_64-pc-linux-gnu
(argv1) start_date: 1992050100
(argv2) exptid: hq0f
(argv3) unique_member_id: 0009
(argv4) batchid: 961
(argv5) wuid: 12177653
(argv6) fclen: 123
(argv7) app_name: oifs_43r3_ps
(argv8) nthreads: 1
Working directory is: /var/lib/boinc-client/slots/9
Project directory is: /var/lib/boinc-client/projects/climateprediction.net/
app name: oifs_43r3_ps
version: 1.05
Location of temp folder: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12177653
..mkdir for temp folder for results failed
Copying: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip to: /var/lib/boinc-client/slots/9/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc-client/slots/9/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_a1fcf8a5b64521898051a80058ab32b3 to: /var/lib/boinc-client/slots/9/oifs_43r3_ps_0009_1992050100_123_961_12177653.zip
Unzipping the namelist zip file: /var/lib/boinc-client/slots/9/oifs_43r3_ps_0009_1992050100_123_961_12177653.zip
ic_ancil_file: ic_ancil_12177653
ifsdata_file: ifsdata_12177653
climate_data_file: clim_data_12177653
horiz_resolution: 159
vert_resolution: 60
grid_type: l_2
upload_interval: 24
utstep: 3600
nfrres: restart dump frequency (steps) 24
Copying IC ancils from: ../../projects/climateprediction.net/jf_9a5d01b7017216030a390d6462d7d2f6 to: /var/lib/boinc-client/slots/9/ic_ancil_12177653.zip
Unzipping the IC ancils zip file: /var/lib/boinc-client/slots/9/ic_ancil_12177653.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_1a1280b7d69ea1eaef8e189a2a2f7d62 to: /var/lib/boinc-client/slots/9/ifsdata/ifsdata_12177653.zip
Unzipping the ifsdata_zip file: /var/lib/boinc-client/slots/9/ifsdata/ifsdata_12177653.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc-client/slots/9/159l_2/clim_data_12177653.zip
Unzipping the climate data zip file: /var/lib/boinc-client/slots/9/159l_2/clim_data_12177653.zip
Checking for progress XML file: /var/lib/boinc-client/slots/9/progress_file_12177653.xml
Opened progress file ok : /var/lib/boinc-client/slots/9/progress_file_12177653.xml
-- Model is restarting --
Adjusting last_iter, 552, to previous model restart step.
Creating progress file: /var/lib/boinc-client/slots/9/progress_file_12177653.xml
last_cpu_time: 27286
upload_file_number: 22
last_iter: 551
last_upload: 1900800
model_completed: 0
total_length_of_simulation: 10627200
result_base_name: oifs_43r3_ps_0009_1992050100_123_961_12177653_1_r721985956
The child process has been launched with process id: 651766
Executing the command: /var/lib/boinc-client/slots/9/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2091] fp = 0x3129fe0
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.000] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.000] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.000] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at (nil))
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230118:234536:1674085536:0.002] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
ABORT! 1 !! *** WAVE MODEL HAS ABORTED
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230120:121835:1674217115:131579.793] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3843] 1721 MB (maxheap), 3359 MB (maxrss), 0 MB (maxstack)
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230120:121835:1674217115:131579.793] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : MASTER
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230120:121835:1674217115:131579.793] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT0
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230120:121835:1674217115:131579.793] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT1
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230120:121835:1674217115:131579.793] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT2
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230120:121835:1674217115:131579.793] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT3
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230120:121835:1674217115:131579.793] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT4
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230120:121835:1674217115:131579.793] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : WVCOUPLE
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230120:121835:1674217115:131579.794] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : WAVEMDL
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230120:121835:1674217115:131579.794] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : INITMDL
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230120:121835:1674217115:131579.794] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : USERIN
SDL_TRACEBACK: Calling INTEL_TRBK, THRD = 1
Process 1 thread 1 calling linux_trbk from intel_trbk()
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:240] Backtrace(s) for program '/var/lib/boinc-client/slots/9/oifs_43r3_model.exe' : sigcontextptr=0x7ffcf45e7420
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:277] Backtrace (size = 19) with addr2line-cmd
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:302] /usr/bin/addr2line -fs -e '/var/lib/boinc-client/slots/9/oifs_43r3_model.exe' 0x839273 0x839879 0x8c2957 0x81d89e 0x15f8aca 0x15d51c8 0x166b07f 0x15d7ff0 0x8711e0 0x7e0165 0x55e4ad 0x5471c9 0x546412 0x545e28 0x40708d 0x40220f 0x4021a2 0x1dca390 0x40206e
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [0]: [0x839273] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [1]: [0x839879] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [2]: [0x8c2957] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [3]: [0x81d89e] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [4]: [0x15f8aca] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [5]: [0x15d51c8] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [6]: [0x166b07f] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [7]: [0x15d7ff0] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [8]: [0x8711e0] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [9]: [0x7e0165] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [10]: [0x55e4ad] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [11]: [0x5471c9] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [12]: [0x546412] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [13]: [0x545e28] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [14]: [0x40708d] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [15]: [0x40220f] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [16]: [0x4021a2] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [17]: [0x1dca390] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [18]: [0x40206e] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:370] End of backtrace(s)
SDL_TRACEBACK: Done INTEL_TRBK, THRD = 1
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230120:121838:1674217118:131582.271] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1538] Received signal#6 (SIGABRT) :: 1721MB (heap), 3359MB (maxrss), 0MB (maxstack), 0 (paging), nsigs = 1
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230120:121838:1674217118:131582.271] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1542] Also activating Harakiri-alarm (SIGALRM=14) to expire after 500s elapsed to prevent hangs, nsigs = 1
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230120:121838:1674217118:131582.271] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1544] Harakiri signal handler 'signal_harakiri' for signal#14 (SIGALRM) installed at 0x81f3c0 (old at (nil))
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230120:121838:1674217118:131582.271] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1617] Signal#6 was caused by unrecognized si_code [memaddr=0x770009f1f6], nsigs = 1
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230120:121838:1674217118:131582.312] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1686] Starting DrHook backtrace for signal#6, nsigs = 1
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230120:121839:1674217119:131583.313] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1734] DrHook backtrace done for signal#6, nsigs = 1
[EC_DRHOOK:Debian12:1:1:651766:651766] [20230120:121839:1674217119:131583.313] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1785] Calling previous signal handler at 0x1ce8bb0 for signal#6, nsigs = 1
forrtl: error (76): Abort trap signal
Image PC Routine Line Source
oifs_43r3_model.e 0000000001CE916B Unknown Unknown Unknown
oifs_43r3_model.e 000000000081FFF1 Unknown Unknown Unknown
oifs_43r3_model.e 0000000001DC9090 Unknown Unknown Unknown
oifs_43r3_model.e 0000000001DC903B Unknown Unknown Unknown
oifs_43r3_model.e 00000000008C2880 sdl_mod_mp_sdl_sr 100 sdl_mod.F90
oifs_43r3_model.e 000000000081D8AF abor1_ 41 abor1.F90
oifs_43r3_model.e 00000000015F8ACA mpabort_ 50 mpabort.F
oifs_43r3_model.e 00000000015D51C8 abort1_ 44 abort1.F
oifs_43r3_model.e 000000000166B07F userin_ 553 userin.F
oifs_43r3_model.e 00000000015D7FF0 initmdl_ 452 initmdl.F
oifs_43r3_model.e 00000000008711E0 wavemdl_ 393 wavemdl.F
oifs_43r3_model.e 00000000007E0165 wvcouple_ 260 wvcouple.F90
oifs_43r3_model.e 000000000055E4AD cnt4_ 1283 cnt4.F90
oifs_43r3_model.e 00000000005471C9 cnt3_ 267 cnt3.F90
oifs_43r3_model.e 0000000000546412 cnt2_ 88 cnt2.F90
oifs_43r3_model.e 0000000000545E28 cnt1_ 92 cnt1.F90
oifs_43r3_model.e 000000000040708D cnt0_ 146 cnt0.F90
oifs_43r3_model.e 000000000040220F MAIN__ 96 master.F90
oifs_43r3_model.e 00000000004021A2 Unknown Unknown Unknown
oifs_43r3_model.e 0000000001DCA390 Unknown Unknown Unknown
oifs_43r3_model.e 000000000040206E Unknown Unknown Unknown
The child process terminated with status: 0
..Failed, model did not complete successfully
</stderr_txt>
]]>
