<core_client_version>7.20.2</core_client_version>
<![CDATA[
<message>
Disk usage limit exceeded</message>
<stderr_txt>
glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:steve-System-Product-Name:1:1:2560:2560] [20230105:114810:1672940890:0.000] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:steve-System-Product-Name:1:1:2560:2560] [20230105:114810:1672940890:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:2560:2560] [20230105:114810:1672940890:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:2560:2560] [20230105:114810:1672940890:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:steve-System-Product-Name:1:1:2560:2560] [20230105:114810:1672940890:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:2560:2560] [20230105:114810:1672940890:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:2560:2560] [20230105:114810:1672940890:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:2560:2560] [20230105:114810:1672940890:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:steve-System-Product-Name:1:1:2560:2560] [20230105:114810:1672940890:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at 0x1)
[EC_DRHOOK:steve-System-Product-Name:1:1:2560:2560] [20230105:114810:1672940890:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:steve-System-Product-Name:1:1:2560:2560] [20230105:114810:1672940890:0.001] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
11:49:22 STEP 1056 H=1056:00 +CPU= 66.530
11:49:58 STEP 1057 H=1057:00 +CPU= 34.503
11:50:26 STEP 1058 H=1058:00 +CPU= 27.068
11:51:07 STEP 1059 H=1059:00 +CPU= 39.890
11:51:38 STEP 1060 H=1060:00 +CPU= 30.951
11:52:10 STEP 1061 H=1061:00 +CPU= 31.594
11:52:56 STEP 1062 H=1062:00 +CPU= 45.246
11:53:31 STEP 1063 H=1063:00 +CPU= 33.414
11:54:01 STEP 1064 H=1064:00 +CPU= 29.361
11:54:53 STEP 1065 H=1065:00 +CPU= 51.846
11:55:30 STEP 1066 H=1066:00 +CPU= 36.388
11:56:05 STEP 1067 H=1067:00 +CPU= 35.301
11:57:06 STEP 1068 H=1068:00 +CPU= 59.416
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001068
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001068
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001068
11:57:41 STEP 1069 H=1069:00 +CPU= 34.659
11:58:19 STEP 1070 H=1070:00 +CPU= 37.889
11:59:16 STEP 1071 H=1071:00 +CPU= 56.041
11:59:53 STEP 1072 H=1072:00 +CPU= 36.441
12:00:29 STEP 1073 H=1073:00 +CPU= 36.158
12:01:27 STEP 1074 H=1074:00 +CPU= 57.037
12:02:03 STEP 1075 H=1075:00 +CPU= 35.668
12:02:41 STEP 1076 H=1076:00 +CPU= 38.162
12:03:38 STEP 1077 H=1077:00 +CPU= 55.829
12:04:15 STEP 1078 H=1078:00 +CPU= 36.550
12:04:53 STEP 1079 H=1079:00 +CPU= 36.920
12:05:42 STEP 1080 H=1080:00 +CPU= 48.223
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001080
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001080
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001080
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001056
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001056
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001056
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001068
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001068
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001068
Zipping up the intermediate file: /home/steve/projects/climateprediction.net/oifs_43r3_ps_0559_2017050100_123_986_12203203_0_r1724738220_44.zip
Uploading the intermediate file: upload_file_44.zip
12:06:11 STEP 1081 H=1081:00 +CPU= 28.478
12:06:46 STEP 1082 H=1082:00 +CPU= 34.169
12:07:35 STEP 1083 H=1083:00 +CPU= 48.607
12:07:55 STEP 1084 H=1084:00 +CPU= 19.790
12:08:32 STEP 1085 H=1085:00 +CPU= 36.595
12:09:23 STEP 1086 H=1086:00 +CPU= 50.087
12:09:55 STEP 1087 H=1087:00 +CPU= 32.399
12:10:32 STEP 1088 H=1088:00 +CPU= 36.106
12:11:28 STEP 1089 H=1089:00 +CPU= 55.800
12:12:05 STEP 1090 H=1090:00 +CPU= 36.908
12:12:42 STEP 1091 H=1091:00 +CPU= 36.816
12:13:44 STEP 1092 H=1092:00 +CPU= 61.192
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001092
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001092
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001092
12:14:21 STEP 1093 H=1093:00 +CPU= 36.174
12:14:58 STEP 1094 H=1094:00 +CPU= 37.412
12:15:54 STEP 1095 H=1095:00 +CPU= 55.397
12:16:32 STEP 1096 H=1096:00 +CPU= 37.498
12:17:09 STEP 1097 H=1097:00 +CPU= 36.839
12:18:07 STEP 1098 H=1098:00 +CPU= 56.726
12:18:44 STEP 1099 H=1099:00 +CPU= 37.179
12:19:25 STEP 1100 H=1100:00 +CPU= 37.786
12:20:09 STEP 1101 H=1101:00 +CPU= 42.315
12:20:38 STEP 1102 H=1102:00 +CPU= 28.795
12:21:14 STEP 1103 H=1103:00 +CPU= 35.523
12:22:01 STEP 1104 H=1104:00 +CPU= 45.745
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001104
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001104
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001104
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001080
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001080
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001080
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001092
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001092
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001092
Zipping up the intermediate file: /home/steve/projects/climateprediction.net/oifs_43r3_ps_0559_2017050100_123_986_12203203_0_r1724738220_45.zip
Uploading the intermediate file: upload_file_45.zip
12:22:29 STEP 1105 H=1105:00 +CPU= 28.060
12:23:07 STEP 1106 H=1106:00 +CPU= 37.262
12:24:03 STEP 1107 H=1107:00 +CPU= 54.675
12:24:34 STEP 1108 H=1108:00 +CPU= 30.273
12:25:05 STEP 1109 H=1109:00 +CPU= 30.221
12:26:02 STEP 1110 H=1110:00 +CPU= 56.881
12:26:38 STEP 1111 H=1111:00 +CPU= 36.091
12:27:16 STEP 1112 H=1112:00 +CPU= 37.281
12:28:14 STEP 1113 H=1113:00 +CPU= 57.092
12:28:51 STEP 1114 H=1114:00 +CPU= 36.581
12:29:27 STEP 1115 H=1115:00 +CPU= 36.193
12:30:30 STEP 1116 H=1116:00 +CPU= 61.963
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001116
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001116
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001116
12:31:06 STEP 1117 H=1117:00 +CPU= 35.913
12:31:43 STEP 1118 H=1118:00 +CPU= 37.242
12:32:41 STEP 1119 H=1119:00 +CPU= 56.896
12:33:17 STEP 1120 H=1120:00 +CPU= 36.262
12:33:54 STEP 1121 H=1121:00 +CPU= 36.067
12:34:52 STEP 1122 H=1122:00 +CPU= 57.363
12:35:28 STEP 1123 H=1123:00 +CPU= 36.145
12:36:05 STEP 1124 H=1124:00 +CPU= 36.945
12:37:02 STEP 1125 H=1125:00 +CPU= 56.010
12:37:39 STEP 1126 H=1126:00 +CPU= 37.126
12:38:17 STEP 1127 H=1127:00 +CPU= 37.586
12:39:24 STEP 1128 H=1128:00 +CPU= 65.780
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001128
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001128
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001128
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001104
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001104
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001104
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001116
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001116
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001116
Zipping up the intermediate file: /home/steve/projects/climateprediction.net/oifs_43r3_ps_0559_2017050100_123_986_12203203_0_r1724738220_46.zip
Uploading the intermediate file: upload_file_46.zip
12:40:00 STEP 1129 H=1129:00 +CPU= 35.932
12:40:38 STEP 1130 H=1130:00 +CPU= 37.123
12:41:34 STEP 1131 H=1131:00 +CPU= 55.508
12:42:10 STEP 1132 H=1132:00 +CPU= 36.278
12:42:48 STEP 1133 H=1133:00 +CPU= 36.867
12:43:44 STEP 1134 H=1134:00 +CPU= 56.232
12:44:21 STEP 1135 H=1135:00 +CPU= 36.229
12:44:58 STEP 1136 H=1136:00 +CPU= 36.842
12:45:54 STEP 1137 H=1137:00 +CPU= 55.282
12:46:32 STEP 1138 H=1138:00 +CPU= 37.299
12:47:09 STEP 1139 H=1139:00 +CPU= 36.602
12:48:13 STEP 1140 H=1140:00 +CPU= 63.953
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001140
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001140
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001140
12:48:50 STEP 1141 H=1141:00 +CPU= 36.769
12:49:30 STEP 1142 H=1142:00 +CPU= 38.760
12:50:28 STEP 1143 H=1143:00 +CPU= 58.034
12:51:05 STEP 1144 H=1144:00 +CPU= 36.905
12:51:43 STEP 1145 H=1145:00 +CPU= 37.471
12:52:43 STEP 1146 H=1146:00 +CPU= 59.100
12:53:20 STEP 1147 H=1147:00 +CPU= 36.585
12:53:59 STEP 1148 H=1148:00 +CPU= 38.774
12:54:57 STEP 1149 H=1149:00 +CPU= 57.470
12:55:36 STEP 1150 H=1150:00 +CPU= 38.267
12:56:16 STEP 1151 H=1151:00 +CPU= 39.571
12:57:25 STEP 1152 H=1152:00 +CPU= 68.646
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001152
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001152
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001152
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001128
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001128
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001128
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001140
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001140
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001140
Zipping up the intermediate file: /home/steve/projects/climateprediction.net/oifs_43r3_ps_0559_2017050100_123_986_12203203_0_r1724738220_47.zip
Uploading the intermediate file: upload_file_47.zip
12:58:02 STEP 1153 H=1153:00 +CPU= 36.051
12:58:41 STEP 1154 H=1154:00 +CPU= 38.855
12:59:40 STEP 1155 H=1155:00 +CPU= 58.195
13:00:18 STEP 1156 H=1156:00 +CPU= 37.534
13:00:57 STEP 1157 H=1157:00 +CPU= 37.886
13:01:57 STEP 1158 H=1158:00 +CPU= 59.353
13:02:33 STEP 1159 H=1159:00 +CPU= 36.581
13:03:13 STEP 1160 H=1160:00 +CPU= 39.232
13:04:12 STEP 1161 H=1161:00 +CPU= 58.066
13:04:49 STEP 1162 H=1162:00 +CPU= 37.281
13:05:28 STEP 1163 H=1163:00 +CPU= 37.964
13:06:34 STEP 1164 H=1164:00 +CPU= 65.296
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001164
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001164
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001164
13:07:09 STEP 1165 H=1165:00 +CPU= 35.259
13:07:48 STEP 1166 H=1166:00 +CPU= 38.381
13:08:47 STEP 1167 H=1167:00 +CPU= 58.365
13:09:24 STEP 1168 H=1168:00 +CPU= 37.089
13:10:03 STEP 1169 H=1169:00 +CPU= 38.555
13:11:03 STEP 1170 H=1170:00 +CPU= 59.421
13:11:40 STEP 1171 H=1171:00 +CPU= 35.996
13:12:20 STEP 1172 H=1172:00 +CPU= 39.398
13:13:18 STEP 1173 H=1173:00 +CPU= 57.660
13:13:56 STEP 1174 H=1174:00 +CPU= 38.076
13:14:36 STEP 1175 H=1175:00 +CPU= 38.697
13:15:45 STEP 1176 H=1176:00 +CPU= 68.460
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001176
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001176
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001176
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001152
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001152
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001152
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001164
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001164
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001164
Zipping up the intermediate file: /home/steve/projects/climateprediction.net/oifs_43r3_ps_0559_2017050100_123_986_12203203_0_r1724738220_48.zip
Uploading the intermediate file: upload_file_48.zip
13:16:21 STEP 1177 H=1177:00 +CPU= 36.101
13:17:01 STEP 1178 H=1178:00 +CPU= 39.047
13:18:01 STEP 1179 H=1179:00 +CPU= 59.518
13:18:39 STEP 1180 H=1180:00 +CPU= 37.635
13:19:17 STEP 1181 H=1181:00 +CPU= 37.391
13:20:18 STEP 1182 H=1182:00 +CPU= 60.819
13:20:56 STEP 1183 H=1183:00 +CPU= 36.850
13:21:36 STEP 1184 H=1184:00 +CPU= 39.937
13:22:35 STEP 1185 H=1185:00 +CPU= 58.125
13:23:13 STEP 1186 H=1186:00 +CPU= 37.829
13:23:52 STEP 1187 H=1187:00 +CPU= 38.739
13:24:57 STEP 1188 H=1188:00 +CPU= 64.639
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001188
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001188
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001188
13:25:34 STEP 1189 H=1189:00 +CPU= 36.623
13:26:13 STEP 1190 H=1190:00 +CPU= 38.758
13:27:13 STEP 1191 H=1191:00 +CPU= 58.359
13:27:49 STEP 1192 H=1192:00 +CPU= 36.532
13:28:28 STEP 1193 H=1193:00 +CPU= 37.898
13:29:29 STEP 1194 H=1194:00 +CPU= 60.335
13:30:06 STEP 1195 H=1195:00 +CPU= 36.910
13:30:45 STEP 1196 H=1196:00 +CPU= 38.762
13:31:44 STEP 1197 H=1197:00 +CPU= 58.556
13:32:21 STEP 1198 H=1198:00 +CPU= 35.809
13:32:55 STEP 1199 H=1199:00 +CPU= 33.392
Quit request received from BOINC client, ending the child process
(argv0) ../../projects/climateprediction.net/oifs_43r3_ps_1.05_x86_64-pc-linux-gnu
(argv1) start_date: 2017050100
(argv2) exptid: hq0f
(argv3) unique_member_id: 0559
(argv4) batchid: 986
(argv5) wuid: 12203203
(argv6) fclen: 123
(argv7) app_name: oifs_43r3_ps
(argv8) nthreads: 1
Working directory is: /home/steve/slots/20
Project directory is: /home/steve/projects/climateprediction.net/
app name: oifs_43r3_ps
version: 1.05
Location of temp folder: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203
..mkdir for temp folder for results failed
Copying: /home/steve/projects/climateprediction.net/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip to: /home/steve/slots/20/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /home/steve/slots/20/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_965dd0baaa1c381ffa151d8e00706c95 to: /home/steve/slots/20/oifs_43r3_ps_0559_2017050100_123_986_12203203.zip
Unzipping the namelist zip file: /home/steve/slots/20/oifs_43r3_ps_0559_2017050100_123_986_12203203.zip
ic_ancil_file: ic_ancil_12203203
ifsdata_file: ifsdata_12203203
climate_data_file: clim_data_12203203
horiz_resolution: 159
vert_resolution: 60
grid_type: l_2
upload_interval: 24
utstep: 3600
nfrres: restart dump frequency (steps) 24
Copying IC ancils from: ../../projects/climateprediction.net/jf_84a18eb2951c08c62807224e96149cda to: /home/steve/slots/20/ic_ancil_12203203.zip
Unzipping the IC ancils zip file: /home/steve/slots/20/ic_ancil_12203203.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_3bfe179b2e3f9b6e43044ac07c404be9 to: /home/steve/slots/20/ifsdata/ifsdata_12203203.zip
Unzipping the ifsdata_zip file: /home/steve/slots/20/ifsdata/ifsdata_12203203.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /home/steve/slots/20/159l_2/clim_data_12203203.zip
Unzipping the climate data zip file: /home/steve/slots/20/159l_2/clim_data_12203203.zip
Checking for progress XML file: /home/steve/slots/20/progress_file_12203203.xml
Opened progress file ok : /home/steve/slots/20/progress_file_12203203.xml
-- Model is restarting --
Adjusting last_iter, 1200, to previous model restart step.
Creating progress file: /home/steve/slots/20/progress_file_12203203.xml
last_cpu_time: 55542
upload_file_number: 49
last_iter: 1199
last_upload: 4233600
model_completed: 0
total_length_of_simulation: 10627200
result_base_name: oifs_43r3_ps_0559_2017050100_123_986_12203203_0_r1724738220
The child process has been launched with process id: 2963
Executing the command: /home/steve/slots/20/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2091] fp = 0x3129fe0
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at 0x1)
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:steve-System-Product-Name:1:1:2963:2963] [20230105:133432:1672947272:0.001] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
13:35:44 STEP 1200 H=1200:00 +CPU= 64.649
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001200
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001200
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001200
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001176
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001176
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001176
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001188
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001188
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001188
Zipping up the intermediate file: /home/steve/projects/climateprediction.net/oifs_43r3_ps_0559_2017050100_123_986_12203203_0_r1724738220_49.zip
Uploading the intermediate file: upload_file_49.zip
13:36:19 STEP 1201 H=1201:00 +CPU= 33.701
13:36:58 STEP 1202 H=1202:00 +CPU= 36.594
13:37:40 STEP 1203 H=1203:00 +CPU= 42.330
13:38:18 STEP 1204 H=1204:00 +CPU= 36.790
13:38:55 STEP 1205 H=1205:00 +CPU= 37.414
13:39:55 STEP 1206 H=1206:00 +CPU= 59.085
13:40:31 STEP 1207 H=1207:00 +CPU= 35.616
13:41:11 STEP 1208 H=1208:00 +CPU= 39.201
13:42:09 STEP 1209 H=1209:00 +CPU= 57.020
13:42:47 STEP 1210 H=1210:00 +CPU= 37.576
13:43:26 STEP 1211 H=1211:00 +CPU= 38.463
13:44:31 STEP 1212 H=1212:00 +CPU= 64.445
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001212
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001212
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001212
13:45:08 STEP 1213 H=1213:00 +CPU= 36.332
13:45:48 STEP 1214 H=1214:00 +CPU= 39.452
13:46:47 STEP 1215 H=1215:00 +CPU= 57.990
13:47:24 STEP 1216 H=1216:00 +CPU= 37.093
13:48:03 STEP 1217 H=1217:00 +CPU= 38.201
13:49:02 STEP 1218 H=1218:00 +CPU= 58.727
13:49:39 STEP 1219 H=1219:00 +CPU= 36.319
13:50:18 STEP 1220 H=1220:00 +CPU= 39.020
13:51:18 STEP 1221 H=1221:00 +CPU= 58.384
13:51:56 STEP 1222 H=1222:00 +CPU= 38.031
13:52:36 STEP 1223 H=1223:00 +CPU= 39.218
13:53:34 STEP 1224 H=1224:00 +CPU= 57.563
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001224
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001224
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001224
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001200
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001200
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001200
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001212
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001212
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001212
Zipping up the intermediate file: /home/steve/projects/climateprediction.net/oifs_43r3_ps_0559_2017050100_123_986_12203203_0_r1724738220_50.zip
Uploading the intermediate file: upload_file_50.zip
13:54:09 STEP 1225 H=1225:00 +CPU= 34.293
13:54:47 STEP 1226 H=1226:00 +CPU= 36.842
13:55:44 STEP 1227 H=1227:00 +CPU= 53.975
13:56:20 STEP 1228 H=1228:00 +CPU= 35.580
13:56:56 STEP 1229 H=1229:00 +CPU= 35.553
13:57:49 STEP 1230 H=1230:00 +CPU= 52.245
13:58:20 STEP 1231 H=1231:00 +CPU= 30.660
13:58:53 STEP 1232 H=1232:00 +CPU= 33.060
13:59:44 STEP 1233 H=1233:00 +CPU= 51.137
14:00:21 STEP 1234 H=1234:00 +CPU= 36.441
14:00:58 STEP 1235 H=1235:00 +CPU= 36.579
14:02:01 STEP 1236 H=1236:00 +CPU= 61.763
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001236
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001236
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001236
14:02:38 STEP 1237 H=1237:00 +CPU= 37.249
14:03:18 STEP 1238 H=1238:00 +CPU= 39.265
14:04:16 STEP 1239 H=1239:00 +CPU= 57.130
14:04:54 STEP 1240 H=1240:00 +CPU= 37.947
14:05:32 STEP 1241 H=1241:00 +CPU= 37.923
14:06:27 STEP 1242 H=1242:00 +CPU= 54.067
14:06:59 STEP 1243 H=1243:00 +CPU= 31.724
14:07:35 STEP 1244 H=1244:00 +CPU= 34.961
14:08:24 STEP 1245 H=1245:00 +CPU= 46.540
14:08:44 STEP 1246 H=1246:00 +CPU= 20.112
14:09:16 STEP 1247 H=1247:00 +CPU= 30.320
14:10:19 STEP 1248 H=1248:00 +CPU= 61.779
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001248
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001248
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001248
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001224
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001224
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001224
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001236
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001236
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001236
Zipping up the intermediate file: /home/steve/projects/climateprediction.net/oifs_43r3_ps_0559_2017050100_123_986_12203203_0_r1724738220_51.zip
Uploading the intermediate file: upload_file_51.zip
14:10:42 STEP 1249 H=1249:00 +CPU= 22.731
14:11:09 STEP 1250 H=1250:00 +CPU= 27.488
14:11:53 STEP 1251 H=1251:00 +CPU= 43.335
14:12:23 STEP 1252 H=1252:00 +CPU= 29.660
14:12:46 STEP 1253 H=1253:00 +CPU= 22.929
14:13:30 STEP 1254 H=1254:00 +CPU= 44.017
14:14:05 STEP 1255 H=1255:00 +CPU= 34.102
14:14:42 STEP 1256 H=1256:00 +CPU= 37.103
14:15:38 STEP 1257 H=1257:00 +CPU= 55.760
14:16:15 STEP 1258 H=1258:00 +CPU= 36.531
14:16:52 STEP 1259 H=1259:00 +CPU= 35.945
14:17:54 STEP 1260 H=1260:00 +CPU= 61.417
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001260
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001260
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001260
14:18:30 STEP 1261 H=1261:00 +CPU= 35.745
14:19:07 STEP 1262 H=1262:00 +CPU= 37.236
14:20:04 STEP 1263 H=1263:00 +CPU= 56.324
14:20:41 STEP 1264 H=1264:00 +CPU= 36.545
14:21:18 STEP 1265 H=1265:00 +CPU= 36.158
14:22:15 STEP 1266 H=1266:00 +CPU= 57.428
14:22:52 STEP 1267 H=1267:00 +CPU= 35.867
14:23:29 STEP 1268 H=1268:00 +CPU= 36.672
14:24:26 STEP 1269 H=1269:00 +CPU= 56.555
14:25:03 STEP 1270 H=1270:00 +CPU= 36.810
14:25:41 STEP 1271 H=1271:00 +CPU= 37.227
14:26:48 STEP 1272 H=1272:00 +CPU= 66.033
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001272
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001272
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001272
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001248
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001248
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001248
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001260
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001260
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001260
Zipping up the intermediate file: /home/steve/projects/climateprediction.net/oifs_43r3_ps_0559_2017050100_123_986_12203203_0_r1724738220_52.zip
Uploading the intermediate file: upload_file_52.zip
14:27:24 STEP 1273 H=1273:00 +CPU= 35.950
14:28:02 STEP 1274 H=1274:00 +CPU= 37.518
14:28:58 STEP 1275 H=1275:00 +CPU= 55.852
14:29:36 STEP 1276 H=1276:00 +CPU= 36.821
14:30:12 STEP 1277 H=1277:00 +CPU= 35.995
14:31:09 STEP 1278 H=1278:00 +CPU= 56.655
14:31:45 STEP 1279 H=1279:00 +CPU= 35.688
14:32:23 STEP 1280 H=1280:00 +CPU= 36.867
14:33:20 STEP 1281 H=1281:00 +CPU= 56.091
14:33:57 STEP 1282 H=1282:00 +CPU= 37.197
14:34:33 STEP 1283 H=1283:00 +CPU= 35.613
14:35:35 STEP 1284 H=1284:00 +CPU= 61.519
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001284
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001284
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001284
14:36:12 STEP 1285 H=1285:00 +CPU= 35.992
Quit request received from BOINC client, ending the child process
..Failed, model did not complete successfully
(argv0) ../../projects/climateprediction.net/oifs_43r3_ps_1.05_x86_64-pc-linux-gnu
(argv1) start_date: 2017050100
(argv2) exptid: hq0f
(argv3) unique_member_id: 0559
(argv4) batchid: 986
(argv5) wuid: 12203203
(argv6) fclen: 123
(argv7) app_name: oifs_43r3_ps
(argv8) nthreads: 1
Working directory is: /home/steve/slots/20
Project directory is: /home/steve/projects/climateprediction.net/
app name: oifs_43r3_ps
version: 1.05
Location of temp folder: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203
..mkdir for temp folder for results failed
Copying: /home/steve/projects/climateprediction.net/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip to: /home/steve/slots/20/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /home/steve/slots/20/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_965dd0baaa1c381ffa151d8e00706c95 to: /home/steve/slots/20/oifs_43r3_ps_0559_2017050100_123_986_12203203.zip
Unzipping the namelist zip file: /home/steve/slots/20/oifs_43r3_ps_0559_2017050100_123_986_12203203.zip
ic_ancil_file: ic_ancil_12203203
ifsdata_file: ifsdata_12203203
climate_data_file: clim_data_12203203
horiz_resolution: 159
vert_resolution: 60
grid_type: l_2
upload_interval: 24
utstep: 3600
nfrres: restart dump frequency (steps) 24
Copying IC ancils from: ../../projects/climateprediction.net/jf_84a18eb2951c08c62807224e96149cda to: /home/steve/slots/20/ic_ancil_12203203.zip
Unzipping the IC ancils zip file: /home/steve/slots/20/ic_ancil_12203203.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_3bfe179b2e3f9b6e43044ac07c404be9 to: /home/steve/slots/20/ifsdata/ifsdata_12203203.zip
Unzipping the ifsdata_zip file: /home/steve/slots/20/ifsdata/ifsdata_12203203.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /home/steve/slots/20/159l_2/clim_data_12203203.zip
Unzipping the climate data zip file: /home/steve/slots/20/159l_2/clim_data_12203203.zip
Checking for progress XML file: /home/steve/slots/20/progress_file_12203203.xml
Opened progress file ok : /home/steve/slots/20/progress_file_12203203.xml
-- Model is restarting --
Adjusting last_iter, 1286, to previous model restart step.
Creating progress file: /home/steve/slots/20/progress_file_12203203.xml
last_cpu_time: 59193
upload_file_number: 53
last_iter: 1271
last_upload: 4579200
model_completed: 0
total_length_of_simulation: 10627200
result_base_name: oifs_43r3_ps_0559_2017050100_123_986_12203203_0_r1724738220
The child process has been launched with process id: 3184
Executing the command: /home/steve/slots/20/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2091] fp = 0x3129fe0
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.001] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.001] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.001] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at 0x1)
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:steve-System-Product-Name:1:1:3184:3184] [20230105:143745:1672951065:0.001] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
14:38:55 STEP 1272 H=1272:00 +CPU= 62.933
14:39:32 STEP 1273 H=1273:00 +CPU= 36.699
14:40:10 STEP 1274 H=1274:00 +CPU= 36.906
14:40:56 STEP 1275 H=1275:00 +CPU= 45.105
14:41:27 STEP 1276 H=1276:00 +CPU= 29.947
14:41:51 STEP 1277 H=1277:00 +CPU= 24.304
14:42:33 STEP 1278 H=1278:00 +CPU= 40.970
14:42:53 STEP 1279 H=1279:00 +CPU= 19.872
14:43:20 STEP 1280 H=1280:00 +CPU= 27.349
14:44:07 STEP 1281 H=1281:00 +CPU= 46.357
14:44:29 STEP 1282 H=1282:00 +CPU= 21.577
14:44:59 STEP 1283 H=1283:00 +CPU= 30.057
14:46:01 STEP 1284 H=1284:00 +CPU= 60.872
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001284
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001284
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001284
14:46:31 STEP 1285 H=1285:00 +CPU= 29.482
14:46:59 STEP 1286 H=1286:00 +CPU= 28.176
14:47:52 STEP 1287 H=1287:00 +CPU= 52.256
14:48:22 STEP 1288 H=1288:00 +CPU= 29.869
14:48:52 STEP 1289 H=1289:00 +CPU= 29.998
14:49:47 STEP 1290 H=1290:00 +CPU= 54.136
14:50:23 STEP 1291 H=1291:00 +CPU= 35.653
14:50:59 STEP 1292 H=1292:00 +CPU= 36.232
14:51:54 STEP 1293 H=1293:00 +CPU= 54.390
14:52:32 STEP 1294 H=1294:00 +CPU= 37.392
14:53:09 STEP 1295 H=1295:00 +CPU= 36.668
14:54:13 STEP 1296 H=1296:00 +CPU= 63.142
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001296
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001296
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001296
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001272
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001272
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001272
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001284
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001284
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001284
Zipping up the intermediate file: /home/steve/projects/climateprediction.net/oifs_43r3_ps_0559_2017050100_123_986_12203203_0_r1724738220_53.zip
Uploading the intermediate file: upload_file_53.zip
14:54:49 STEP 1297 H=1297:00 +CPU= 36.445
14:55:25 STEP 1298 H=1298:00 +CPU= 35.431
14:56:20 STEP 1299 H=1299:00 +CPU= 54.010
14:56:57 STEP 1300 H=1300:00 +CPU= 37.054
14:57:32 STEP 1301 H=1301:00 +CPU= 34.601
14:58:28 STEP 1302 H=1302:00 +CPU= 54.955
14:59:05 STEP 1303 H=1303:00 +CPU= 36.672
14:59:40 STEP 1304 H=1304:00 +CPU= 34.838
15:00:35 STEP 1305 H=1305:00 +CPU= 54.227
15:01:12 STEP 1306 H=1306:00 +CPU= 37.032
15:01:47 STEP 1307 H=1307:00 +CPU= 34.733
15:02:47 STEP 1308 H=1308:00 +CPU= 59.129
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001308
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001308
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001308
15:03:24 STEP 1309 H=1309:00 +CPU= 36.841
15:04:01 STEP 1310 H=1310:00 +CPU= 36.982
15:04:56 STEP 1311 H=1311:00 +CPU= 53.612
15:05:33 STEP 1312 H=1312:00 +CPU= 37.315
15:06:08 STEP 1313 H=1313:00 +CPU= 34.296
15:07:03 STEP 1314 H=1314:00 +CPU= 54.796
15:07:40 STEP 1315 H=1315:00 +CPU= 36.336
15:08:15 STEP 1316 H=1316:00 +CPU= 35.000
15:09:10 STEP 1317 H=1317:00 +CPU= 54.216
15:09:47 STEP 1318 H=1318:00 +CPU= 36.495
15:10:25 STEP 1319 H=1319:00 +CPU= 37.302
15:11:25 STEP 1320 H=1320:00 +CPU= 59.670
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001320
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001320
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001320
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001296
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001296
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001296
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001308
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001308
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001308
Zipping up the intermediate file: /home/steve/projects/climateprediction.net/oifs_43r3_ps_0559_2017050100_123_986_12203203_0_r1724738220_54.zip
Uploading the intermediate file: upload_file_54.zip
15:12:01 STEP 1321 H=1321:00 +CPU= 36.126
15:12:38 STEP 1322 H=1322:00 +CPU= 36.265
15:13:34 STEP 1323 H=1323:00 +CPU= 55.658
15:14:13 STEP 1324 H=1324:00 +CPU= 38.260
15:14:49 STEP 1325 H=1325:00 +CPU= 35.467
15:15:46 STEP 1326 H=1326:00 +CPU= 56.747
15:16:24 STEP 1327 H=1327:00 +CPU= 37.224
15:17:00 STEP 1328 H=1328:00 +CPU= 35.462
15:17:56 STEP 1329 H=1329:00 +CPU= 56.223
15:18:35 STEP 1330 H=1330:00 +CPU= 38.032
15:19:11 STEP 1331 H=1331:00 +CPU= 35.839
15:20:14 STEP 1332 H=1332:00 +CPU= 62.213
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001332
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001332
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001332
15:20:51 STEP 1333 H=1333:00 +CPU= 37.385
15:21:29 STEP 1334 H=1334:00 +CPU= 37.248
15:22:25 STEP 1335 H=1335:00 +CPU= 55.903
15:23:04 STEP 1336 H=1336:00 +CPU= 38.008
15:23:40 STEP 1337 H=1337:00 +CPU= 35.590
15:24:37 STEP 1338 H=1338:00 +CPU= 56.368
15:25:14 STEP 1339 H=1339:00 +CPU= 36.691
15:25:50 STEP 1340 H=1340:00 +CPU= 36.127
15:26:46 STEP 1341 H=1341:00 +CPU= 55.359
15:27:24 STEP 1342 H=1342:00 +CPU= 37.701
15:28:02 STEP 1343 H=1343:00 +CPU= 37.429
15:29:09 STEP 1344 H=1344:00 +CPU= 66.440
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001344
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001344
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001344
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001320
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001320
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001320
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001332
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001332
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001332
Zipping up the intermediate file: /home/steve/projects/climateprediction.net/oifs_43r3_ps_0559_2017050100_123_986_12203203_0_r1724738220_55.zip
Uploading the intermediate file: upload_file_55.zip
15:29:46 STEP 1345 H=1345:00 +CPU= 36.814
15:30:23 STEP 1346 H=1346:00 +CPU= 36.601
15:31:20 STEP 1347 H=1347:00 +CPU= 55.983
15:31:59 STEP 1348 H=1348:00 +CPU= 37.789
15:32:35 STEP 1349 H=1349:00 +CPU= 36.007
15:33:32 STEP 1350 H=1350:00 +CPU= 56.465
15:34:10 STEP 1351 H=1351:00 +CPU= 37.582
15:34:47 STEP 1352 H=1352:00 +CPU= 36.921
15:35:43 STEP 1353 H=1353:00 +CPU= 55.084
15:36:22 STEP 1354 H=1354:00 +CPU= 38.799
15:36:59 STEP 1355 H=1355:00 +CPU= 36.103
15:38:00 STEP 1356 H=1356:00 +CPU= 60.819
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001356
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001356
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001356
15:38:38 STEP 1357 H=1357:00 +CPU= 37.828
15:39:16 STEP 1358 H=1358:00 +CPU= 37.091
15:40:12 STEP 1359 H=1359:00 +CPU= 55.246
15:40:50 STEP 1360 H=1360:00 +CPU= 37.888
15:41:26 STEP 1361 H=1361:00 +CPU= 36.040
Quit request received from BOINC client, ending the child process
(argv0) ../../projects/climateprediction.net/oifs_43r3_ps_1.05_x86_64-pc-linux-gnu
(argv1) start_date: 2017050100
(argv2) exptid: hq0f
(argv3) unique_member_id: 0559
(argv4) batchid: 986
(argv5) wuid: 12203203
(argv6) fclen: 123
(argv7) app_name: oifs_43r3_ps
(argv8) nthreads: 1
Working directory is: /home/steve/slots/20
Project directory is: /home/steve/projects/climateprediction.net/
app name: oifs_43r3_ps
version: 1.05
Location of temp folder: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203
..mkdir for temp folder for results failed
Copying: /home/steve/projects/climateprediction.net/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip to: /home/steve/slots/20/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /home/steve/slots/20/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_965dd0baaa1c381ffa151d8e00706c95 to: /home/steve/slots/20/oifs_43r3_ps_0559_2017050100_123_986_12203203.zip
Unzipping the namelist zip file: /home/steve/slots/20/oifs_43r3_ps_0559_2017050100_123_986_12203203.zip
ic_ancil_file: ic_ancil_12203203
ifsdata_file: ifsdata_12203203
climate_data_file: clim_data_12203203
horiz_resolution: 159
vert_resolution: 60
grid_type: l_2
upload_interval: 24
utstep: 3600
nfrres: restart dump frequency (steps) 24
Copying IC ancils from: ../../projects/climateprediction.net/jf_84a18eb2951c08c62807224e96149cda to: /home/steve/slots/20/ic_ancil_12203203.zip
Unzipping the IC ancils zip file: /home/steve/slots/20/ic_ancil_12203203.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_3bfe179b2e3f9b6e43044ac07c404be9 to: /home/steve/slots/20/ifsdata/ifsdata_12203203.zip
Unzipping the ifsdata_zip file: /home/steve/slots/20/ifsdata/ifsdata_12203203.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /home/steve/slots/20/159l_2/clim_data_12203203.zip
Unzipping the climate data zip file: /home/steve/slots/20/159l_2/clim_data_12203203.zip
Checking for progress XML file: /home/steve/slots/20/progress_file_12203203.xml
Opened progress file ok : /home/steve/slots/20/progress_file_12203203.xml
-- Model is restarting --
Adjusting last_iter, 1362, to previous model restart step.
Creating progress file: /home/steve/slots/20/progress_file_12203203.xml
last_cpu_time: 63012
upload_file_number: 56
last_iter: 1343
last_upload: 4838400
model_completed: 0
total_length_of_simulation: 10627200
result_base_name: oifs_43r3_ps_0559_2017050100_123_986_12203203_0_r1724738220
The child process has been launched with process id: 3370
Executing the command: /home/steve/slots/20/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2091] fp = 0x3129fe0
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at 0x1)
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:steve-System-Product-Name:1:1:3370:3370] [20230105:154319:1672954999:0.001] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
15:44:32 STEP 1344 H=1344:00 +CPU= 65.236
15:45:09 STEP 1345 H=1345:00 +CPU= 35.305
15:45:43 STEP 1346 H=1346:00 +CPU= 34.343
15:46:40 STEP 1347 H=1347:00 +CPU= 56.223
15:47:17 STEP 1348 H=1348:00 +CPU= 36.743
15:47:54 STEP 1349 H=1349:00 +CPU= 36.732
15:48:51 STEP 1350 H=1350:00 +CPU= 56.634
15:49:29 STEP 1351 H=1351:00 +CPU= 36.758
15:50:07 STEP 1352 H=1352:00 +CPU= 37.624
15:51:06 STEP 1353 H=1353:00 +CPU= 58.505
15:51:44 STEP 1354 H=1354:00 +CPU= 38.110
15:52:22 STEP 1355 H=1355:00 +CPU= 37.430
15:53:27 STEP 1356 H=1356:00 +CPU= 64.327
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001356
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001356
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001356
15:54:05 STEP 1357 H=1357:00 +CPU= 37.683
15:54:43 STEP 1358 H=1358:00 +CPU= 38.057
15:55:42 STEP 1359 H=1359:00 +CPU= 57.863
15:56:20 STEP 1360 H=1360:00 +CPU= 38.502
15:56:58 STEP 1361 H=1361:00 +CPU= 36.965
15:57:57 STEP 1362 H=1362:00 +CPU= 59.107
15:58:35 STEP 1363 H=1363:00 +CPU= 37.706
15:59:14 STEP 1364 H=1364:00 +CPU= 38.134
16:00:12 STEP 1365 H=1365:00 +CPU= 57.780
16:00:51 STEP 1366 H=1366:00 +CPU= 38.409
16:01:31 STEP 1367 H=1367:00 +CPU= 39.983
16:02:40 STEP 1368 H=1368:00 +CPU= 67.867
Moving to projects directory: /home/steve/slots/20/ICMGGhq0f+001368
Moving to projects directory: /home/steve/slots/20/ICMSHhq0f+001368
Moving to projects directory: /home/steve/slots/20/ICMUAhq0f+001368
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001344
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001344
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001344
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMGGhq0f+001356
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMSHhq0f+001356
Adding to the zip: /home/steve/projects/climateprediction.net/oifs_43r3_ps_12203203/ICMUAhq0f+001356
Zipping up the intermediate file: /home/steve/projects/climateprediction.net/oifs_43r3_ps_0559_2017050100_123_986_12203203_0_r1724738220_56.zip
Uploading the intermediate file: upload_file_56.zip
16:03:18 STEP 1369 H=1369:00 +CPU= 37.427
16:03:57 STEP 1370 H=1370:00 +CPU= 38.336
16:04:56 STEP 1371 H=1371:00 +CPU= 58.383
Abort request received from BOINC client, ending the child process
..Failed, model did not complete successfully
</stderr_txt>
]]>
