From c1d22b08537405bc8065ad42db96a54df44ecfbf Mon Sep 17 00:00:00 2001 From: dhyan1272 Date: Thu, 9 Jan 2025 10:08:16 -0800 Subject: [PATCH 01/42] New Regualization method with dual cell triangle involved --- src/pmpo_MPMesh_assembly.hpp | 29 ++++++++++++++++++++++++++--- src/pmpo_c.cpp | 36 ++++++++++++++++++++++++++++++++++++ src/pmpo_c.h | 5 +++++ src/pmpo_fortran.f90 | 28 ++++++++++++++++++++++++++++ src/pmpo_mesh.cpp | 4 ++++ src/pmpo_mesh.hpp | 7 +++++++ src/pmpo_utils.hpp | 12 ++++++++++++ 7 files changed, 118 insertions(+), 3 deletions(-) diff --git a/src/pmpo_MPMesh_assembly.hpp b/src/pmpo_MPMesh_assembly.hpp index e5dedb5..c577506 100644 --- a/src/pmpo_MPMesh_assembly.hpp +++ b/src/pmpo_MPMesh_assembly.hpp @@ -111,6 +111,9 @@ void MPMesh::assemblyVtx1() { p_mesh->fillMeshField(numVtx, numEntries, 0.0); auto meshField = p_mesh->getMeshField(); + //Dual Element Area for Regularization + auto dual_triangle_area=p_mesh->getMeshField(); + //Material Points auto mpData = p_MPs->getData(); auto weight = p_MPs->getData(); @@ -146,6 +149,7 @@ void MPMesh::assemblyVtx1() { }; p_MPs->parallel_for(assemble, "assembly"); + bool regularize=true; //Solve Ax=b for each vertex Kokkos::View VtxCoeffs("VtxCoeffs", p_mesh->getNumVertices()); Kokkos::parallel_for("solving Ax=b", numVtx, KOKKOS_LAMBDA(const int vtx){ @@ -154,13 +158,32 @@ void MPMesh::assemblyVtx1() { Vec4d v2 = {VtxMatrices(vtx,2,0), VtxMatrices(vtx,2,1), VtxMatrices(vtx,2,2), VtxMatrices(vtx,2,3)}; Vec4d v3 = {VtxMatrices(vtx,3,0), VtxMatrices(vtx,3,1), VtxMatrices(vtx,3,2), VtxMatrices(vtx,3,3)}; - Matrix4d A = {v0,v1,v2,v3}; - double A_trace = A.trace(); + Matrix4d A = {v0,v1,v2,v3}; Matrix4d A_regularized = {v0, v1, v2, v3}; + + //Method 1 of rgularization + //Need some kind of option to choose regularization method + /* + double A_trace = A.trace(); A_regularized.addToDiag(A_trace*1e-8); - + */ + + //Method 2 of regularization + double mScale=1.0; + if(regularize){ + mScale=sqrt(dual_triangle_area(vtx,0)); + A_regularized.scaleFirstRowAndColumn(mScale); + double regParam=0.0*EPSILON*VtxMatrices(vtx,0,0) + VtxMatrices(vtx,1,1) + VtxMatrices(vtx,2,2) + VtxMatrices(vtx,3,3); + A_regularized.addToDiag(regParam); + } double coeff[vec4d_nEntries]={0.0, 0.0, 0.0, 0.0}; CholeskySolve4d_UnitRHS(A_regularized, coeff); + + // Undo scaling + coeff[0]=coeff[0]*mScale*mScale; + coeff[1]=coeff[1]*mScale; + coeff[2]=coeff[2]*mScale; + coeff[3]=coeff[3]*mScale; for (int i=0; ip_mesh; + + //check the size + PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); + + //copy the host array to the device + auto dualArea = p_mesh->getMeshField(); + auto h_dualArea = Kokkos::create_mirror_view(dualArea); + for(int i=0; ip_mesh; + + //check the size + PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); + + //copy the device to host + auto dualArea = p_mesh->getMeshField(); + auto h_dualArea = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), dualArea); + for(int i=0; i @brief set the polympo dual mesh triangle area + !> @param mpmesh(in/out) MPMesh object + !> @param nVertices(in) length of array in, use for assertion + !> @param Array(in) 1D array of area of dual triangle elements + !--------------------------------------------------------------------------- + subroutine polympo_setMeshDualTriangleArea(mpMesh, nVertices, areaTriangle) & + bind(C, NAME='polympo_setMeshDualTriangleArea_f') + use :: iso_c_binding + type(c_ptr), value :: mpMesh + integer(c_int), value :: nVertices + type(c_ptr), intent(in), value :: areaTriangle + end subroutine + !--------------------------------------------------------------------------- + !> @brief get the polympo mesh dual mesh triangle area + !> @param mpmesh(in/out) MPMesh object + !> @param nVetices(in) length of array in, use for assertion + !> @param Array(in/out) 1D array of area of dual triangle elements + !--------------------------------------------------------------------------- + subroutine polympo_getMeshDualTriangleArea(mpMesh, nVertices, areaTriangle) & + bind(C, NAME='polympo_getMeshDualTriangleArea_f') + use :: iso_c_binding + type(c_ptr), value :: mpMesh + integer(c_int), value :: nVertices + type(c_ptr), value :: areaTriangle + end subroutine + !--------------------------------------------------------------------------- !> @brief set the vertices velocity from a host array !> @param mpmesh(in/out) MPMesh object diff --git a/src/pmpo_mesh.cpp b/src/pmpo_mesh.cpp index cdf9b28..9232ad1 100644 --- a/src/pmpo_mesh.cpp +++ b/src/pmpo_mesh.cpp @@ -41,6 +41,10 @@ namespace polyMPO{ auto vtxRotLatLonIncrMapEntry = meshFields2TypeAndString.at(MeshF_RotLatLonIncr); PMT_ALWAYS_ASSERT(vtxRotLatLonIncrMapEntry.first == MeshFType_VtxBased); vtxRotLatLonIncr_ = MeshFView(vtxRotLatLonIncrMapEntry.second,numVtxs_); + + auto dualTriangleAreaEntry = meshFields2TypeAndString.at(MeshF_DualTriangleArea); + PMT_ALWAYS_ASSERT(dualTriangleAreaEntry.first == MeshFType_VtxBased); + dualTriangleArea_ = MeshFView(dualTriangleAreaEntry.second,numVtxs_); } void Mesh::setMeshElmBasedFieldSize(){ diff --git a/src/pmpo_mesh.hpp b/src/pmpo_mesh.hpp index 3b91dc3..2058e13 100644 --- a/src/pmpo_mesh.hpp +++ b/src/pmpo_mesh.hpp @@ -20,6 +20,7 @@ enum MeshFieldIndex{ MeshF_VtxCoords, MeshF_VtxRotLat, MeshF_ElmCenterXYZ, + MeshF_DualTriangleArea, MeshF_Vel, MeshF_VtxMass, MeshF_ElmMass, @@ -38,6 +39,7 @@ template struct meshFieldToType; template <> struct meshFieldToType < MeshF_VtxCoords > { using type = Kokkos::View; }; template <> struct meshFieldToType < MeshF_VtxRotLat > { using type = DoubleView; }; template <> struct meshFieldToType < MeshF_ElmCenterXYZ > { using type = Kokkos::View; }; +template <> struct meshFieldToType < MeshF_DualTriangleArea > { using type = Kokkos::View; }; template <> struct meshFieldToType < MeshF_Vel > { using type = Kokkos::View; }; template <> struct meshFieldToType < MeshF_VtxMass > { using type = Kokkos::View; }; template <> struct meshFieldToType < MeshF_ElmMass > { using type = Kokkos::View; }; @@ -55,6 +57,7 @@ const std::map vtxCoords_; MeshFView vtxRotLat_; MeshFView elmCenterXYZ_; + MeshFView dualTriangleArea_; MeshFView vtxVel_; MeshFView vtxMass_; MeshFView elmMass_; @@ -181,6 +185,9 @@ auto Mesh::getMeshField(){ else if constexpr (index==MeshF_ElmCenterXYZ){ return elmCenterXYZ_; } + else if constexpr (index==MeshF_DualTriangleArea){ + return dualTriangleArea_; + } else if constexpr (index==MeshF_Vel){ return vtxVel_; } diff --git a/src/pmpo_utils.hpp b/src/pmpo_utils.hpp index b9d5f7d..4bd4f88 100644 --- a/src/pmpo_utils.hpp +++ b/src/pmpo_utils.hpp @@ -274,6 +274,18 @@ class Matrix4d { data_[i][i]=data_[i][i]+eps; } } + + KOKKOS_INLINE_FUNCTION + void scaleFirstRowAndColumn(double factor) { + // Scale the first row + for (int j = 0; j < 4; j++) { + data_[0][j] *= factor; + } + // Scale the first column + for (int i = 0; i < 4; i++) { + data_[i][0] *= factor; + } + } }; From eb482e8c51e412577fd463554f23a2f0af953c48 Mon Sep 17 00:00:00 2001 From: dhyan1272 Date: Mon, 10 Feb 2025 07:56:44 -0800 Subject: [PATCH 02/42] Scaling and regularization --- log_MP | 6863 ++++++++++++++++++++++++++ src/pmpo_MPMesh_assembly.hpp | 69 +- src/pmpo_utils.hpp | 15 +- test/testFortranMPReconstruction.f90 | 3 +- 4 files changed, 6918 insertions(+), 32 deletions(-) create mode 100644 log_MP diff --git a/log_MP b/log_MP new file mode 100644 index 0000000..39918fb --- /dev/null +++ b/log_MP @@ -0,0 +1,6863 @@ +netcdf ic_slotted_cylinder_2562 { +dimensions: + nCells = 2562 ; + nVertices = 5120 ; + nCategories = 1 ; + ONE = 1 ; +variables: + double uVelocity(nVertices) ; + double vVelocity(nVertices) ; + double iceAreaCell(nCells) ; + double iceVolumeCell(nCells) ; + double iceAreaCategory(nCells, nCategories, ONE) ; + double iceVolumeCategory(nCells, nCategories, ONE) ; +data: + + uVelocity = 3.51971928314205, 3.47512234552054, 3.39605364751305, + 3.39605364776838, 3.47512234595256, 3.39605364415435, 3.47512234253748, + 3.51971928024815, 3.47512234234787, 3.39605364424297, 0.154357464161867, + 0.154357464418338, 0.154357464376962, 0.154357464500569, + 0.154357464373758, 3.39605364751305, 3.47512234552054, 3.51971928314207, + 3.47512234595257, 3.39605364776839, 3.39605364415435, 3.39605364424296, + 3.47512234234787, 3.51971928024815, 3.47512234253748, 3.39605365600016, + 3.47512235331092, 3.51971929051509, 3.47512235377981, 3.39605365628955, + 3.39605364650079, 3.4751223448265, 3.51971928232674, 3.47512234510602, + 3.3960536468386, 3.3960536465008, 3.39605364683861, 3.47512234510602, + 3.51971928232674, 3.4751223448265, 0.154357464161846, 0.154357464373768, + 0.154357464500558, 0.154357464376972, 0.154357464418328, + 3.47512234965195, 3.51971928719354, 3.47512235073297, 3.39605365277294, + 3.3960536517862, 3.39605365277296, 3.47512235073298, 3.51971928719355, + 3.47512234965196, 3.39605365178622, 3.39605365600017, 3.39605365628957, + 3.47512235377982, 3.51971929051509, 3.47512235331093, 3.33632040348494, + 3.3707523490026, 3.33632040282711, 3.23195483390815, 3.19170730176026, + 3.23195483460147, 3.86052719315628, 3.85735494265993, 3.8600357605879, + 3.86052719312186, 3.85735494257026, 3.86003576053382, 3.85735494241866, + 3.86052719307657, 3.86003576063311, 3.85735494279346, 3.86052719322088, + 3.86003576043921, 3.33632039673443, 3.37075234295757, 3.33632039667594, + 3.23195482756115, 3.19170729467244, 3.23195482762904, 2.14741220053217, + 2.14741220011114, 2.03504821319991, 1.91331103281558, 1.9133110333481, + 2.03504821501541, 3.85735494318127, 3.86003576090522, 3.86052719293465, + 3.85735494202528, 3.86003576029375, 3.86052719337439, 3.86052719290206, + 3.86003576093899, 3.85735494327473, 3.86052719341129, 3.86003576023198, + 3.85735494194176, 3.3363203952616, 3.23195482587916, 3.1917072928257, + 3.23195482556806, 3.33632039492412, 3.37075234133105, 2.14741219436931, + 2.14741219388141, 2.03504820739424, 1.91331102618635, 1.91331102674549, + 2.03504820792936, 1.91331103481011, 2.03504821492811, 2.14741220272273, + 2.14741220392819, 2.03504821887084, 1.9133110359939, 2.14741221165298, + 2.03504822395832, 1.91331104400562, 1.91331104358003, 2.03504822645895, + 2.14741221130944, 2.14741219417938, 2.03504820734298, 1.9133110261006, + 1.91331102602604, 2.03504820752655, 2.14741219418535, 3.86052719322088, + 3.85735494279346, 3.86003576063311, 3.86052719307657, 3.85735494241866, + 3.86003576043921, 3.86052719312186, 3.8600357605879, 3.85735494265993, + 3.86052719315628, 3.86003576053382, 3.85735494257026, 3.3363204028271, + 3.37075234900259, 3.33632040348493, 3.23195483460147, 3.19170730176026, + 3.23195483390815, 1.9133110333481, 1.91331103281558, 2.03504821319992, + 2.14741220011115, 2.14741220053218, 2.03504821501541, 3.33632039673443, + 3.23195482762903, 3.19170729467243, 3.23195482756114, 3.33632039667594, + 3.37075234295757, 3.86052719293466, 3.86003576090522, 3.85735494318127, + 3.86052719337439, 3.86003576029375, 3.85735494202528, 1.91331102618635, + 2.03504820739425, 2.14741219388141, 2.14741219436931, 2.03504820792935, + 1.9133110267455, 3.33632039492412, 3.23195482556806, 3.19170729282571, + 3.23195482587917, 3.3363203952616, 3.37075234133105, 3.86003576093899, + 3.86052719290206, 3.85735494194176, 3.86003576023198, 3.86052719341129, + 3.85735494327473, 3.23195484163699, 3.19170730967967, 3.23195484186681, + 3.33632041042294, 3.37075235592902, 3.33632041023479, 2.14741221165297, + 2.14741221130942, 2.03504822645893, 1.91331104358002, 1.91331104400561, + 2.03504822395829, 3.86003576034742, 3.86052719332038, 3.85735494301239, + 3.86003576077637, 3.86052719300741, 3.85735494219614, 3.33632039797719, + 3.23195482781526, 3.19170729537129, 3.23195482758278, 3.33632039772401, + 3.3707523434905, 1.91331102602603, 1.91331102610058, 2.03504820734297, + 2.14741219417937, 2.14741219418535, 2.03504820752655, 3.33632039797719, + 3.37075234349048, 3.33632039772401, 3.23195482758276, 3.19170729537129, + 3.23195482781525, 3.85735494301239, 3.86052719332038, 3.86003576034742, + 3.85735494219614, 3.86052719300741, 3.86003576077637, 2.14741220272273, + 2.03504821492812, 1.91331103481012, 1.91331103599391, 2.03504821887085, + 2.1474122039282, 3.19170730967966, 3.23195484163698, 3.33632041023479, + 3.37075235592902, 3.33632041042294, 3.23195484186681, 3.29636636362926, + 3.24773691482299, 3.27150647779161, 3.37084721086564, 3.41435559172517, + 3.39249536885371, 3.76894522317819, 3.7353246005687, 3.70681333198533, + 3.72534591930479, 3.76403894959448, 3.78623820284751, 3.7353245999012, + 3.7689452226303, 3.78623820219564, 3.76403894866395, 3.72534591818006, + 3.70681333098936, 3.27150647403733, 3.24773691135723, 3.29636636079297, + 3.39249536603282, 3.41435558865646, 3.37084720718277, 2.95756490679779, + 2.87690638401446, 2.77685515752924, 2.77685515780303, 2.87690638510949, + 2.95756490706125, 2.87690638022213, 2.95756490249572, 2.95756490235291, + 2.87690637954507, 2.7768551526422, 2.77685515289343, 3.29636635933764, + 3.24773691023361, 3.27150647312524, 3.37084720638359, 3.41435558767133, + 3.39249536475112, 3.70681332896657, 3.72534591679101, 3.76403894695546, + 3.78623820082457, 3.76894522059566, 3.73532459820695, 3.73532459794603, + 3.768945220452, 3.7862382005649, 3.76403894665404, 3.7253459162681, + 3.70681332863849, 3.29636635776033, 3.39249536309733, 3.41435558602935, + 3.37084720445461, 3.27150647116826, 3.24773690820345, 0.903438045650939, + 1.03257039861662, 1.16261876910387, 1.16261876965589, 1.03257039872572, + 0.903438046431373, 0.903438053592724, 1.03257040536273, 1.16261877767811, + 1.16261877906432, 1.03257041067264, 0.903438055183172, 0.903438061183353, + 1.03257041680396, 1.16261878592089, 1.16261878643253, 1.03257041488004, + 0.903438061661399, 1.03257040616398, 0.903438053635351, + 0.903438055078504, 1.03257040919781, 1.16261877847409, 1.16261877734117, + 1.03257039949571, 0.903438046662163, 0.903438047790002, 1.0325704004215, + 1.16261877103789, 1.1626187701149, 3.29636636079297, 3.24773691135723, + 3.27150647403733, 3.37084720718277, 3.41435558865647, 3.39249536603282, + 3.76894522263029, 3.7353245999012, 3.70681333098936, 3.72534591818006, + 3.76403894866395, 3.78623820219564, 3.7353246005687, 3.76894522317818, + 3.78623820284752, 3.76403894959447, 3.7253459193048, 3.70681333198533, + 3.247736914823, 3.29636636362927, 3.39249536885371, 3.41435559172518, + 3.37084721086565, 3.27150647779162, 2.95756490679779, 2.95756490706126, + 2.87690638510949, 2.77685515780302, 2.77685515752923, 2.87690638401446, + 3.72534591679101, 3.70681332896656, 3.73532459820695, 3.76894522059565, + 3.78623820082457, 3.76403894695546, 3.2477369102336, 3.29636635933764, + 3.39249536475111, 3.41435558767132, 3.37084720638358, 3.27150647312524, + 2.95756490249571, 2.87690638022212, 2.77685515289343, 2.77685515264219, + 2.87690637954508, 2.95756490235289, 3.29636635776033, 3.24773690820345, + 3.27150647116826, 3.37084720445462, 3.41435558602936, 3.39249536309733, + 3.73532459794603, 3.70681332863849, 3.7253459162681, 3.76403894665404, + 3.78623820056489, 3.768945220452, 3.24773692441733, 3.29636637269177, + 3.39249537736691, 3.41435559992311, 3.37084721945235, 3.27150648687924, + 2.8769063948919, 2.95756491772741, 2.95756491780715, 2.8769063964159, + 2.77685516890115, 2.77685516892971, 3.29636636938179, 3.24773692030793, + 3.27150648241272, 3.37084721516382, 3.4143555962887, 3.39249537413924, + 3.70681333568204, 3.7253459226604, 3.76403895224453, 3.78623820548746, + 3.76894522614054, 3.73532460428986, 3.70681333674755, 3.73532460502842, + 3.76894522673302, 3.78623820620996, 3.76403895324676, 3.7253459239122, + 3.70681333043198, 3.72534591774692, 3.7640389480836, 3.78623820181339, + 3.76894522210539, 3.73532459949979, 3.73532460009807, 3.76894522282055, + 3.78623820247427, 3.76403894917696, 3.72534591863172, 3.70681333147836, + 3.24773691120959, 3.29636636054265, 3.39249536635517, 3.41435558881604, + 3.37084720765732, 3.27150647379344, 2.77685515429861, 2.8769063816351, + 2.95756490432179, 2.95756490447969, 2.87690638155258, 2.77685515435747, + 3.29636635980591, 3.24773691052675, 3.27150647296846, 3.37084720645808, + 3.4143555878722, 3.39249536540832, 3.27150647296846, 3.24773691052675, + 3.29636635980591, 3.39249536540832, 3.41435558787219, 3.37084720645808, + 2.9575649043218, 2.8769063816351, 2.77685515429862, 2.77685515435748, + 2.87690638155259, 2.95756490447969, 3.29636636054265, 3.24773691120959, + 3.27150647379342, 3.37084720765732, 3.41435558881603, 3.39249536635517, + 3.76894522282056, 3.73532460009807, 3.70681333147836, 3.72534591863172, + 3.76403894917696, 3.78623820247427, 3.70681333043198, 3.73532459949979, + 3.76894522210539, 3.78623820181339, 3.76403894808361, 3.72534591774692, + 0.903438046662163, 1.03257039949573, 1.16261877011491, 1.16261877103791, + 1.03257040042149, 0.903438047790018, 0.903438053635359, 1.03257040616398, + 1.16261877734117, 1.16261877847409, 1.03257040919781, 0.903438055078513, + 1.03257041680396, 0.903438061183345, 0.903438061661404, 1.03257041488003, + 1.16261878643252, 1.16261878592087, 1.03257040536272, 0.903438053592724, + 0.903438055183179, 1.03257041067264, 1.16261877906432, 1.1626187776781, + 0.903438045650941, 0.90343804643137, 1.03257039872571, 1.1626187696559, + 1.16261876910389, 1.03257039861661, 3.24773692218824, 3.29636637051913, + 3.39249537553824, 3.41435559839331, 3.37084721805891, 3.27150648520813, + 2.77685516163244, 2.87690638833652, 2.95756491116434, 2.9575649125277, + 2.87690639049826, 2.77685516298305, 3.37084720951275, 3.4143555906345, + 3.39249536880745, 3.29636636297443, 3.24773691395421, 3.27150647558195, + 3.76894522472282, 3.73532460220802, 3.7068133334731, 3.7253459202698, + 3.76403895060766, 3.78623820394614, 3.7253459229499, 3.70681333565846, + 3.73532460382477, 3.76894522585012, 3.78623820545346, 3.76403895252925, + 3.70681333565847, 3.7253459229499, 3.76403895252925, 3.78623820545346, + 3.76894522585012, 3.73532460382476, 3.73532460220802, 3.76894522472282, + 3.78623820394614, 3.76403895060766, 3.7253459202698, 3.7068133334731, + 3.39249536880746, 3.4143555906345, 3.37084720951276, 3.27150647558194, + 3.24773691395422, 3.29636636297443, 2.95756491116436, 2.87690638833653, + 2.77685516163244, 2.77685516298306, 2.87690639049828, 2.95756491252772, + 3.29636637051912, 3.24773692218824, 3.27150648520813, 3.37084721805891, + 3.41435559839331, 3.39249537553824, 3.73532460502842, 3.70681333674755, + 3.7253459239122, 3.76403895324676, 3.78623820620996, 3.76894522673302, + 3.76403895224453, 3.7253459226604, 3.70681333568204, 3.73532460428986, + 3.76894522614055, 3.78623820548746, 3.27150648241272, 3.24773692030795, + 3.2963663693818, 3.39249537413925, 3.4143555962887, 3.37084721516383, + 2.95756491772742, 2.87690639489191, 2.77685516892972, 2.77685516890116, + 2.87690639641591, 2.95756491780717, 3.29636637269178, 3.24773692441733, + 3.27150648687924, 3.37084721945236, 3.41435559992312, 3.39249537736691, + 3.69034136284513, 3.66902586196525, 3.69610022881746, 3.74352818035439, + 3.76062802068365, 3.73904385009254, 3.73904384915359, 3.76062801951332, + 3.74352817885714, 3.69610022697635, 3.66902586039812, 3.69034136170115, + 2.68983921205122, 2.75552219944641, 2.71743080819205, 2.59000888207046, + 2.51344855277885, 2.55900622258916, 2.55900622580187, 2.5134485569339, + 2.59000888740643, 2.71743081296537, 2.75552220330374, 2.68983921494602, + 3.76062801822243, 3.7435281777863, 3.69610022525304, 3.66902585859627, + 3.69034135959205, 3.73904384779614, 3.85429643637894, 3.85870568247999, + 3.86103574461437, 3.85946486675864, 3.86103574461713, 3.85870568246069, + 3.85429643661807, 3.85870568266521, 3.86103574459901, 3.85946486664261, + 3.86103574460411, 3.85870568261925, 3.85870568218824, 3.86103574465829, + 3.85946486702695, 3.86103574466378, 3.85870568216127, 3.8542964358189, + 3.73904384643936, 3.69034135855334, 3.66902585724456, 3.6961002243344, + 3.74352817661265, 3.76062801727404, 3.73904384810876, 3.69034135995964, + 3.66902585911129, 3.69610022588481, 3.74352817832292, 3.76062801862327, + 2.68983920460133, 2.75552219278086, 2.71743080150696, 2.59000887541825, + 2.51344854530942, 2.55900621516455, 2.55900621604943, 2.51344854661861, + 2.59000887703654, 2.71743080274771, 2.75552219367367, 2.68983920527075, + 3.73904384608154, 3.7606280169476, 3.74352817596999, 3.69610022391329, + 3.66902585678567, 3.69034135833747, 1.92822120497328, 1.85158065035756, + 1.71673000672189, 1.62632047729976, 1.71673000829342, 1.8515806516817, + 1.92822119535366, 1.85158064069737, 1.71672999729685, 1.6263204671434, + 1.71672999818415, 1.85158064146224, 3.7606280230259, 3.74352818249984, + 3.69610023151787, 3.66902586514668, 3.6903413662693, 3.73904385281335, + 3.85429643730031, 3.85870568308304, 3.86103574455386, 3.85946486629979, + 3.86103574455475, 3.85870568307874, 3.73904385294298, 3.69034136645886, + 3.66902586540083, 3.69610023185968, 3.74352818274118, 3.76062802319882, + 3.85870568203261, 3.86103574468196, 3.85946486714144, 3.8610357446785, + 3.85870568205915, 3.85429643558444, 3.7390438452, 3.69034135723507, + 3.66902585564614, 3.69610022281084, 3.74352817513519, 3.76062801610617, + 3.73904384571644, 3.76062801671597, 3.74352817595134, 3.69610022373314, + 3.66902585648341, 3.69034135782162, 2.68983920291653, 2.75552219136651, + 2.71743080052465, 2.59000887428552, 2.51344854383711, 2.55900621324726, + 2.55900621230208, 2.51344854275856, 2.59000887285915, 2.7174307990401, + 2.7555221900842, 2.68983920195731, 1.92822119160674, 1.85158063755821, + 1.71672999376142, 1.62632046329999, 1.71672999368156, 1.85158063745382, + 1.85158063947254, 1.71672999611985, 1.62632046481581, 1.71672999490435, + 1.85158063836851, 1.9282211930361, 2.55900621513632, 2.68983920528556, + 2.75552219342873, 2.71743080364367, 2.5900088770865, 2.51344854670601, + 2.55900623044055, 2.51344856098806, 2.59000888952688, 2.71743081648028, + 2.75552220758498, 2.68983922087117, 1.92822121048097, 1.85158065838457, + 1.71673001255179, 1.62632048317417, 1.71673001130163, 1.851580657115, + 2.55900623226872, 2.68983922270568, 2.75552220953349, 2.71743081948777, + 2.59000889252288, 2.51344856394869, 2.55900621345203, 2.51344854401996, + 2.59000887420429, 2.71743080075885, 2.75552219160256, 2.68983920357477, + 3.73904384810876, 3.76062801862328, 3.74352817832292, 3.69610022588482, + 3.66902585911129, 3.69034135995965, 3.85429643637894, 3.8587056824607, + 3.86103574461713, 3.85946486675864, 3.86103574461437, 3.85870568247999, + 3.76062801727404, 3.74352817661265, 3.6961002243344, 3.66902585724455, + 3.69034135855335, 3.73904384643936, 3.8542964358189, 3.85870568216127, + 3.86103574466378, 3.85946486702695, 3.86103574465829, 3.85870568218824, + 3.85870568261926, 3.86103574460411, 3.85946486664261, 3.86103574459901, + 3.85870568266521, 3.85429643661808, 3.73904384915359, 3.69034136170115, + 3.66902586039812, 3.69610022697636, 3.74352817885714, 3.76062801951333, + 3.73904384779614, 3.69034135959206, 3.66902585859628, 3.69610022525305, + 3.7435281777863, 3.76062801822244, 2.55900622580186, 2.68983921494601, + 2.75552220330374, 2.71743081296536, 2.59000888740643, 2.51344855693388, + 2.55900622258917, 2.51344855277885, 2.59000888207047, 2.71743080819205, + 2.75552219944642, 2.68983921205122, 3.73904385009254, 3.76062802068365, + 3.74352818035439, 3.69610022881747, 3.66902586196525, 3.69034136284513, + 1.85158064146224, 1.71672999818415, 1.62632046714338, 1.71672999729685, + 1.85158064069735, 1.92822119535365, 2.55900621604943, 2.68983920527074, + 2.75552219367367, 2.7174308027477, 2.59000887703654, 2.51344854661859, + 1.92822120497329, 1.85158065168169, 1.71673000829342, 1.62632047729975, + 1.71673000672189, 1.85158065035755, 3.73904384608153, 3.69034135833747, + 3.66902585678566, 3.69610022391329, 3.74352817596998, 3.76062801694759, + 2.55900621516454, 2.5134485453094, 2.59000887541824, 2.71743080150694, + 2.75552219278085, 2.68983920460131, 3.85870568307874, 3.86103574455475, + 3.85946486629979, 3.86103574455386, 3.85870568308304, 3.85429643730031, + 3.73904385281335, 3.6903413662693, 3.66902586514668, 3.69610023151787, + 3.74352818249984, 3.7606280230259, 1.71672999368156, 1.62632046330001, + 1.71672999376143, 1.85158063755823, 1.92822119160676, 1.85158063745385, + 2.5590062123021, 2.68983920195732, 2.75552219008421, 2.71743079904012, + 2.59000887285916, 2.51344854275857, 2.55900621324727, 2.51344854383711, + 2.59000887428552, 2.71743080052466, 2.75552219136651, 2.68983920291653, + 3.66902585648342, 3.69610022373314, 3.74352817595135, 3.76062801671596, + 3.73904384571643, 3.69034135782162, 3.7390438452, 3.76062801610617, + 3.74352817513519, 3.69610022281084, 3.66902585564614, 3.69034135723507, + 3.85429643558444, 3.85870568205915, 3.8610357446785, 3.85946486714144, + 3.86103574468196, 3.85870568203261, 3.73904385294298, 3.76062802319882, + 3.74352818274118, 3.69610023185968, 3.66902586540083, 3.69034136645886, + 2.75552220758498, 2.71743081648029, 2.59000888952688, 2.51344856098807, + 2.55900623044056, 2.68983922087118, 2.55900623226872, 2.5134485639487, + 2.59000889252288, 2.71743081948776, 2.75552220953348, 2.68983922270568, + 1.92822121048097, 1.851580657115, 1.71673001130164, 1.62632048317417, + 1.71673001255179, 1.85158065838458, 1.85158063836852, 1.71672999490434, + 1.62632046481582, 1.71672999611984, 1.85158063947254, 1.92822119303609, + 2.55900621345202, 2.68983920357478, 2.75552219160255, 2.71743080075886, + 2.59000887420428, 2.51344854401997, 2.55900621513632, 2.51344854670602, + 2.59000887708649, 2.71743080364368, 2.75552219342872, 2.68983920528556, + 3.31181370257387, 3.35908132346166, 3.44625835690342, 3.4633277441073, + 3.41800761050411, 3.32584574099211, 3.26804799730986, 3.20942955850101, + 3.12164836499262, 3.12164836525515, 3.20942955923946, 3.26804799757476, + 3.35908132450756, 3.31181370406625, 3.32584574291193, 3.41800761247519, + 3.46332774556996, 3.44625835807049, 3.6450147615448, 3.60067298658623, + 3.56009474911326, 3.58034875564143, 3.63825080787157, 3.67297415202974, + 3.56009474835612, 3.60067298625167, 3.64501476122664, 3.67297415146641, + 3.63825080692829, 3.58034875465491, 3.3590813206922, 3.31181370020673, + 3.32584573889264, 3.41800760864817, 3.46332774205432, 3.44625835445029, + 3.56009474573959, 3.58034875259364, 3.63825080461649, 3.67297414943854, + 3.64501475846348, 3.60067298383726, 3.5803487521188, 3.56009474546507, + 3.60067298372279, 3.64501475837662, 3.6729741492274, 3.6382508042955, + 3.31181369907863, 3.35908132025738, 3.44625835374865, 3.46332774127064, + 3.41800760742445, 3.32584573785137, 3.26804799362015, 3.26804799367781, + 3.20942955460651, 3.12164836096841, 3.12164836100006, 3.20942955528946, + 0.513170980391654, 0.390101460425834, 0.39010146214344, + 0.513170982837199, 0.644390362785376, 0.644390361391548, + 0.39010145995051, 0.51317097986375, 0.644390360553818, 0.644390361614694, + 0.513170981309957, 0.390101461351891, 0.390101464472655, + 0.513170985220625, 0.644390367300331, 0.644390369210505, + 0.513170992235016, 0.390101466818805, 0.390101469340862, + 0.513170995446163, 0.644390373865335, 0.644390374285878, + 0.513170994512027, 0.390101469695325, 0.390101464823125, + 0.390101467148519, 0.513170991705255, 0.644390369430307, + 0.644390367590229, 0.513170986531185, 3.60067298658624, 3.64501476154479, + 3.67297415202976, 3.63825080787157, 3.58034875564144, 3.56009474911326, + 3.31181370406625, 3.35908132450757, 3.4462583580705, 3.46332774556998, + 3.4180076124752, 3.32584574291194, 3.20942955850101, 3.26804799730986, + 3.26804799757476, 3.20942955923946, 3.12164836525515, 3.12164836499261, + 3.35908132346166, 3.31181370257388, 3.32584574099211, 3.41800761050411, + 3.46332774410731, 3.44625835690342, 3.56009474835613, 3.58034875465492, + 3.63825080692828, 3.67297415146642, 3.64501476122664, 3.60067298625168, + 3.31181370020672, 3.35908132069219, 3.44625835445029, 3.46332774205431, + 3.41800760864816, 3.32584573889263, 3.26804799362015, 3.20942955528946, + 3.12164836100006, 3.12164836096841, 3.20942955460651, 3.26804799367781, + 3.35908132025737, 3.31181369907862, 3.32584573785136, 3.41800760742444, + 3.46332774127064, 3.44625835374865, 3.56009474546506, 3.5803487521188, + 3.6382508042955, 3.67297414922739, 3.64501475837662, 3.60067298372279, + 3.56009474573959, 3.60067298383726, 3.64501475846347, 3.67297414943854, + 3.63825080461649, 3.58034875259363, 3.20942956821056, 3.26804800673831, + 3.26804800699638, 3.20942956910064, 3.12164837543289, 3.12164837525338, + 3.35908133210083, 3.31181371157333, 3.3258457496641, 3.41800761859748, + 3.46332775179875, 3.4462583648768, 3.56009475520654, 3.58034876125019, + 3.6382508127279, 3.67297415692659, 3.64501476699879, 3.60067299271539, + 3.58034876235721, 3.56009475602123, 3.60067299309049, 3.64501476733615, + 3.67297415755316, 3.63825081373643, 3.35908133326547, 3.44625836615578, + 3.46332775346192, 3.41800762080316, 3.32584575184952, 3.31181371327306, + 3.35908132245567, 3.31181370178286, 3.32584574001317, 3.41800760989976, + 3.4633277433773, 3.44625835626692, 3.56009474779591, 3.58034875408694, + 3.63825080639685, 3.67297415100882, 3.6450147607534, 3.60067298565548, + 3.600672985903, 3.64501476108012, 3.67297415145966, 3.6382508072732, + 3.58034875468341, 3.5600947484059, 3.31181370203029, 3.35908132303664, + 3.44625835680565, 3.46332774403365, 3.41800761064153, 3.32584574069303, + 3.26804799594899, 3.26804799628088, 3.20942955723016, 3.12164836340318, + 3.12164836314393, 3.20942955730212, 3.64501476108012, 3.60067298590299, + 3.5600947484059, 3.58034875468341, 3.63825080727321, 3.67297415145966, + 3.58034875408695, 3.56009474779592, 3.60067298565547, 3.64501476075342, + 3.67297415100882, 3.63825080639685, 3.31181370178285, 3.35908132245566, + 3.44625835626693, 3.4633277433773, 3.41800760989977, 3.32584574001317, + 3.26804799594899, 3.20942955730213, 3.12164836314393, 3.12164836340318, + 3.20942955723017, 3.26804799628088, 3.35908132303663, 3.31181370203028, + 3.32584574069302, 3.41800761064153, 3.46332774403365, 3.44625835680565, + 0.513170979863757, 0.390101459950514, 0.390101461351887, + 0.513170981309937, 0.644390361614679, 0.644390360553805, + 0.390101460425834, 0.513170980391677, 0.644390361391548, + 0.644390362785384, 0.513170982837199, 0.390101462143453, + 0.390101464823112, 0.513170986531194, 0.644390367590255, + 0.64439036943033, 0.513170991705272, 0.390101467148507, + 0.513170995446179, 0.390101469340858, 0.390101469695334, + 0.513170994512023, 0.644390374285886, 0.644390373865327, + 0.39010146447265, 0.390101466818792, 0.513170992235013, + 0.644390369210495, 0.644390367300323, 0.513170985220632, + 3.46332774715531, 3.44625836059314, 3.3590813270382, 3.3118137064222, + 3.32584574385911, 3.41800761373586, 3.64501476467293, 3.60067298969088, + 3.56009475193647, 3.58034875769823, 3.63825080995988, 3.67297415433584, + 3.58034876026232, 3.56009475377362, 3.60067299058148, 3.64501476540208, + 3.67297415576365, 3.63825081214336, 3.31181370995532, 3.35908133015054, + 3.44625836348736, 3.46332775106351, 3.41800761854248, 3.32584574918805, + 3.26804800175343, 3.26804800296036, 3.20942956456826, 3.12164837079488, + 3.12164836947269, 3.2094295631454, 3.60067298969089, 3.64501476467293, + 3.67297415433583, 3.63825080995988, 3.58034875769823, 3.56009475193648, + 3.44625836059316, 3.46332774715532, 3.41800761373587, 3.32584574385913, + 3.31181370642221, 3.35908132703821, 3.26804800175343, 3.20942956314541, + 3.1216483694727, 3.12164837079489, 3.20942956456827, 3.26804800296038, + 3.35908133015054, 3.31181370995533, 3.32584574918805, 3.41800761854248, + 3.46332775106352, 3.44625836348737, 3.56009475377363, 3.58034876026232, + 3.63825081214337, 3.67297415576365, 3.64501476540209, 3.60067299058148, + 3.31181371157334, 3.35908133210083, 3.44625836487681, 3.46332775179875, + 3.41800761859748, 3.3258457496641, 3.26804800673831, 3.20942956821056, + 3.12164837525338, 3.1216483754329, 3.20942956910064, 3.26804800699638, + 3.35908133326547, 3.31181371327306, 3.32584575184952, 3.41800762080316, + 3.46332775346192, 3.44625836615579, 3.56009475602124, 3.58034876235721, + 3.63825081373643, 3.67297415755317, 3.64501476733615, 3.6006729930905, + 3.56009475520653, 3.60067299271539, 3.6450147669988, 3.6729741569266, + 3.63825081272791, 3.58034876125018, 3.08082660801717, 3.04241173682454, + 2.9206912327572, 2.86476252917547, 2.9105012306686, 3.0334125478374, + 2.9105012315398, 2.86476253095996, 2.92069123535957, 3.04241173925898, + 3.08082660957038, 3.03341254865553, 3.35389278326266, 3.38203646367467, + 3.34224676909528, 3.2387419855312, 3.20616415863302, 3.25207929631784, + 3.46488607518047, 3.49924465455745, 3.57603285089435, 3.60145794013974, + 3.57210180618271, 3.49454781110009, 3.57210180569235, 3.60145793920912, + 3.57603284944596, 3.49924465294645, 3.46488607403731, 3.49454781055657, + 3.38203646125342, 3.35389278008579, 3.25207929293554, 3.20616415572886, + 3.2387419834683, 3.34224676714462, 3.83730049868562, 3.81602439422394, + 3.79952374854542, 3.81216850450131, 3.83475040255377, 3.84515172371925, + 3.85524174976196, 3.85898334461991, 3.86098785453913, 3.85897089845091, + 3.8608413352106, 3.85946975252995, 3.84228359978211, 3.82457407797039, + 3.80871340093741, 3.82041520176199, 3.8399968010498, 3.8495135572642, + 3.83730049842567, 3.81602439371894, 3.79952374798096, 3.81216850381735, + 3.83475040220811, 3.84515172343446, 3.85524174961272, 3.85898334449836, + 3.86098785455535, 3.85897089853854, 3.8608413352336, 3.85946975244585, + 3.84228359998145, 3.82457407823984, 3.80871340125699, 3.8204152021239, + 3.83999680125938, 3.84951355742011, 3.86084133519797, 3.85897089845485, + 3.86098785454272, 3.85898334455374, 3.8552417497379, 3.8594697525296, + 3.83475040251148, 3.81216850420118, 3.79952374850035, 3.81602439426909, + 3.8373004988564, 3.84515172372913, 3.82457407890996, 3.84228360050848, + 3.8495135577529, 3.83999680161838, 3.82041520249319, 3.808713401902, + 3.86084133529588, 3.85897089874415, 3.86098785460524, 3.85898334428236, + 3.85524174925527, 3.85946975229149, 3.83475040154957, 3.81216850309196, + 3.79952374711499, 3.81602439313399, 3.83730049793289, 3.84515172301957, + 3.82457407770568, 3.84228359963091, 3.84951355708345, 3.83999680072582, + 3.82041520132951, 3.80871340048797, 3.4648860689349, 3.49924464850608, + 3.57603284532914, 3.60145793511001, 3.5721018009682, 3.49454780548832, + 3.57210180088351, 3.60145793499067, 3.57603284505017, 3.49924464832333, + 3.46488606879208, 3.49454780543662, 3.25207928778574, 3.20616415000751, + 3.23874197774547, 3.34224676160499, 3.38203645632499, 3.35389277511682, + 3.080826601482, 3.04241173027962, 2.92069122612521, 2.86476252177829, + 2.91050122330073, 3.03341254054834, 2.91050122343423, 2.86476252206779, + 2.92069122656977, 3.04241173059118, 3.08082660167768, 3.03341254063997, + 3.35389277555892, 3.38203645662013, 3.34224676181183, 3.23874197798445, + 3.20616415034957, 3.25207928829333, 2.28022860418703, 2.40265427515554, + 2.4295968822255, 2.34121441610718, 2.21188697166056, 2.17953696695939, + 1.9849406019266, 1.88375469162434, 1.75116085122506, 1.70415170279435, + 1.81775135254054, 1.94602812550664, 1.41889854835934, 1.54672376368563, + 1.66983705266571, 1.66983705198026, 1.54672376143888, 1.41889854747539, + 1.81775135569696, 1.704151707568, 1.75116085710719, 1.88375469823445, + 1.98494060702109, 1.94602812971535, 2.21188697610915, 2.34121442127904, + 2.42959688613418, 2.40265427847282, 2.28022860654878, 2.17953697069596, + 2.58069898330844, 2.48250524607293, 2.37040346111973, 2.37040346146522, + 2.48250524752885, 2.58069898360735, 3.83730049722269, 3.81602439252289, + 3.79952374635626, 3.81216850268673, 3.83475040104545, 3.84515172264257, + 3.85524174902513, 3.85898334420375, 3.86098785463037, 3.85897089887936, + 3.86084133536623, 3.85946975217001, 3.80871340338234, 3.82041520419242, + 3.839996802564, 3.84951355844971, 3.84228360113373, 3.82457408008348, + 3.83730050014674, 3.81602439649907, 3.79952375106963, 3.81216850691302, + 3.83475040413067, 3.84515172504442, 3.85524175045754, 3.85898334507543, + 3.86098785446156, 3.85897089801443, 3.86084133508315, 3.85946975293424, + 3.84228359857435, 3.82457407668573, 3.80871339914924, 3.82041520050351, + 3.83999679982684, 3.84951355646953, 3.86084133506967, 3.85897089799313, + 3.86098785445721, 3.8589833450892, 3.85524175050246, 3.85946975295874, + 3.83475040424791, 3.81216850705744, 3.79952375129251, 3.81602439672981, + 3.83730050032849, 3.84515172514695, 3.82457408051469, 3.84228360143838, + 3.84951355865041, 3.8399968028122, 3.82041520452226, 3.80871340383006, + 3.86084133539843, 3.85897089899106, 3.86098785465421, 3.85898334408679, + 3.8552417488472, 3.85946975207691, 3.83475040068896, 3.81216850212871, + 3.79952374582553, 3.81602439205011, 3.83730049695183, 3.84515172235639, + 3.82457407647112, 3.84228359847614, 3.84951355633341, 3.83999679965052, + 3.8204152001564, 3.80871339890026, 3.4648860671801, 3.4992446466888, + 3.57603284364376, 3.60145793367096, 3.57210179959393, 3.49454780394676, + 3.57210179985685, 3.60145793414029, 3.57603284441111, 3.49924464746755, + 3.4648860677802, 3.49454780421399, 3.2520792870555, 3.20616414912086, + 3.2387419765059, 3.34224676063611, 3.38203645550112, 3.35389277464523, + 3.08082659980027, 3.04241172894884, 2.9206912245272, 2.86476251997308, + 2.91050122114396, 3.03341253869673, 2.91050122079906, 2.86476251926956, + 2.92069122352958, 3.04241172788238, 3.08082659914702, 3.033412538331, + 3.35389277309147, 3.38203645440123, 3.34224675965113, 3.23874197560188, + 3.206164147754, 3.25207928564178, 2.28022859870545, 2.40265427033085, + 2.4295968774201, 2.34121441165764, 2.2118869666104, 2.17953696186471, + 1.98494059668979, 1.88375468687305, 1.75116084588048, 1.70415169724174, + 1.81775134649606, 1.94602812016069, 1.54672375502529, 1.41889854029187, + 1.41889854105772, 1.54672375559485, 1.66983704566626, 1.66983704501851, + 1.8177513481593, 1.70415169892245, 1.75116084857611, 1.88375468933316, + 1.9849405991179, 1.94602812170514, 2.21188696886946, 2.3412144138293, + 2.42959687955111, 2.40265427179645, 2.28022860012117, 2.17953696334986, + 2.48250524155944, 2.58069897831205, 2.58069897796797, 2.48250524096618, + 2.37040345532657, 2.37040345574696, 1.70415171207159, 1.81775135925391, + 1.94602813482244, 1.98494061211482, 1.88375470439201, 1.75116086138526, + 1.41889854829708, 1.54672376170737, 1.66983705336537, 1.66983705456452, + 1.54672376632688, 1.41889854955953, 1.81775135280092, 1.70415170224912, + 1.75116085007806, 1.88375469050174, 1.98494060130122, 1.94602812567358, + 2.2118869716462, 2.34121441649983, 2.42959688285938, 2.40265427689687, + 2.28022860569235, 2.17953696790764, 2.4825052492982, 2.5806989870123, + 2.58069898831851, 2.4825052523831, 2.37040346556608, 2.37040346431457, + 2.34121442840181, 2.21188698191546, 2.17953697632402, 2.28022861154443, + 2.40265428451121, 2.42959689251844, 1.81775136502913, 1.70415171757556, + 1.75116086502996, 1.88375470809254, 1.98494061622916, 1.94602814066237, + 2.21188698622573, 2.34121443274437, 2.4295968968027, 2.40265429031028, + 2.28022861731359, 2.17953698224677, 2.58069899496667, 2.48250525726598, + 2.3704034728221, 2.37040347257654, 2.48250525939679, 2.58069899482802, + 2.28022861555448, 2.4026542861926, 2.4295968931892, 2.3412144268598, + 2.21188698258817, 2.17953697788018, 1.98494061271456, 1.88375470218215, + 1.75116086179774, 1.70415171324896, 1.8177513634387, 1.94602813621702, + 1.41889855692285, 1.54672377371729, 1.66983706215624, 1.66983706264065, + 1.54672377126065, 1.41889855743696, 1.54672375422311, 1.41889853979863, + 1.41889853943904, 1.54672375443117, 1.6698370444596, 1.66983704465869, + 1.81775134660655, 1.70415169718831, 1.75116084605761, 1.88375468713567, + 1.98494059721597, 1.94602812044671, 2.21188696754454, 2.34121441266494, + 2.42959687876722, 2.4026542714854, 2.28022859968374, 2.17953696256324, + 2.48250524207868, 2.58069897922181, 2.58069897921054, 2.48250524198014, + 2.37040345608139, 2.37040345611661, 2.28022859950511, 2.40265427129882, + 2.42959687854649, 2.34121441254737, 2.21188696735763, 2.17953696233429, + 1.98494059721609, 1.88375468720816, 1.75116084642749, 1.70415169707677, + 1.81775134667405, 1.94602812022711, 3.85897089845485, 3.86084133519797, + 3.8594697525296, 3.8552417497379, 3.85898334455374, 3.86098785454272, + 3.84228359963091, 3.82457407770568, 3.80871340048796, 3.82041520132951, + 3.83999680072582, 3.84951355708345, 3.83730049793289, 3.81602439313399, + 3.79952374711499, 3.81216850309196, 3.83475040154957, 3.84515172301957, + 3.85524174925527, 3.85898334428236, 3.86098785460524, 3.85897089874415, + 3.86084133529588, 3.85946975229149, 3.84228360050848, 3.82457407890995, + 3.808713401902, 3.82041520249319, 3.83999680161838, 3.84951355775291, + 3.81216850420119, 3.83475040251148, 3.84515172372913, 3.8373004988564, + 3.81602439426909, 3.79952374850035, 3.8608413352336, 3.85897089853854, + 3.86098785455535, 3.85898334449836, 3.85524174961272, 3.85946975244585, + 3.83475040220811, 3.81216850381735, 3.79952374798096, 3.81602439371894, + 3.83730049842566, 3.84515172343446, 3.82457407797039, 3.8422835997821, + 3.8495135572642, 3.8399968010498, 3.82041520176199, 3.80871340093741, + 3.8608413352106, 3.85897089845091, 3.86098785453913, 3.85898334461991, + 3.85524174976196, 3.85946975252995, 3.83475040255377, 3.81216850450131, + 3.79952374854541, 3.81602439422395, 3.83730049868561, 3.84515172371925, + 3.82457407823984, 3.84228359998145, 3.84951355742012, 3.83999680125938, + 3.8204152021239, 3.80871340125699, 3.04241173682453, 3.08082660801718, + 3.0334125478374, 2.9105012306686, 2.86476252917546, 2.92069123275721, + 3.35389278008578, 3.38203646125342, 3.34224676714461, 3.2387419834683, + 3.20616415572886, 3.25207929293555, 3.46488607403731, 3.49924465294645, + 3.57603284944596, 3.60145793920913, 3.57210180569235, 3.49454781055657, + 3.5721018061827, 3.60145794013974, 3.57603285089434, 3.49924465455745, + 3.46488607518046, 3.49454781110009, 3.38203646367467, 3.35389278326265, + 3.25207929631784, 3.20616415863301, 3.2387419855312, 3.34224676909527, + 2.86476253095995, 2.9105012315398, 3.03341254865552, 3.08082660957038, + 3.04241173925897, 2.92069123535956, 2.28022860654879, 2.40265427847283, + 2.42959688613419, 2.34121442127905, 2.21188697610917, 2.17953697069598, + 1.9849406070211, 1.88375469823444, 1.75116085710719, 1.70415170756799, + 1.81775135569698, 1.94602812971534, 1.54672376368563, 1.41889854835936, + 1.41889854747541, 1.54672376143889, 1.66983705198027, 1.66983705266572, + 1.81775135254055, 1.70415170279434, 1.75116085122506, 1.88375469162434, + 1.98494060192661, 1.94602812550664, 2.21188697166056, 2.34121441610718, + 2.4295968822255, 2.40265427515554, 2.28022860418704, 2.17953696695939, + 2.48250524607293, 2.58069898330845, 2.58069898360736, 2.48250524752886, + 2.37040346146523, 2.37040346111974, 3.08082660167768, 3.04241173059116, + 2.92069122656976, 2.86476252206777, 2.91050122343422, 3.03341254063995, + 2.91050122330072, 2.86476252177827, 2.9206912261252, 3.04241173027962, + 3.080826601482, 3.03341254054833, 3.2061641500075, 3.25207928778573, + 3.35389277511681, 3.38203645632499, 3.34224676160499, 3.23874197774547, + 3.46488606879207, 3.49924464832333, 3.57603284505016, 3.60145793499066, + 3.5721018008835, 3.49454780543662, 3.5721018009682, 3.60145793511002, + 3.57603284532914, 3.49924464850608, 3.4648860689349, 3.49454780548833, + 3.38203645662013, 3.35389277555892, 3.25207928829333, 3.20616415034956, + 3.23874197798444, 3.34224676181183, 3.81602439252289, 3.8373004972227, + 3.84515172264257, 3.83475040104545, 3.81216850268673, 3.79952374635626, + 3.82457407668573, 3.84228359857435, 3.84951355646953, 3.83999679982684, + 3.82041520050351, 3.80871339914924, 3.86084133508315, 3.85897089801443, + 3.86098785446156, 3.85898334507543, 3.85524175045754, 3.85946975293424, + 3.83475040413067, 3.81216850691302, 3.79952375106963, 3.81602439649907, + 3.83730050014675, 3.84515172504442, 3.82041520419242, 3.80871340338233, + 3.82457408008348, 3.84228360113373, 3.84951355844971, 3.839996802564, + 3.85898334420375, 3.85524174902513, 3.85946975217001, 3.86084133536623, + 3.85897089887936, 3.86098785463037, 2.40265427033085, 2.28022859870546, + 2.17953696186473, 2.21188696661041, 2.34121441165766, 2.4295968774201, + 2.58069897831205, 2.48250524155943, 2.37040345574696, 2.37040345532657, + 2.48250524096618, 2.58069897796796, 2.28022860012117, 2.40265427179644, + 2.42959687955111, 2.34121441382928, 2.21188696886945, 2.17953696334984, + 1.9849405991179, 1.88375468933315, 1.75116084857611, 1.70415169892243, + 1.81775134815931, 1.94602812170512, 1.41889854029187, 1.54672375502529, + 1.6698370450185, 1.66983704566626, 1.54672375559484, 1.41889854105773, + 1.88375468687307, 1.98494059668979, 1.94602812016069, 1.81775134649606, + 1.70415169724175, 1.75116084588049, 3.08082659914702, 3.04241172788239, + 2.92069122352959, 2.86476251926958, 2.91050122079907, 3.03341253833101, + 2.91050122114398, 2.86476251997311, 2.92069122452722, 3.04241172894886, + 3.08082659980029, 3.03341253869675, 3.20616414912087, 3.25207928705551, + 3.35389277464524, 3.38203645550113, 3.34224676063612, 3.23874197650591, + 3.4648860677802, 3.49924464746755, 3.57603284441111, 3.60145793414029, + 3.57210179985686, 3.49454780421399, 3.57210179959394, 3.60145793367097, + 3.57603284364376, 3.4992446466888, 3.4648860671801, 3.49454780394675, + 3.38203645440123, 3.35389277309147, 3.25207928564178, 3.20616414775401, + 3.23874197560188, 3.34224675965113, 3.83730049695183, 3.81602439205011, + 3.79952374582553, 3.81216850212871, 3.83475040068896, 3.84515172235639, + 3.8552417488472, 3.85898334408679, 3.86098785465421, 3.85897089899106, + 3.86084133539843, 3.85946975207691, 3.84228360143838, 3.82457408051469, + 3.80871340383006, 3.82041520452226, 3.8399968028122, 3.84951355865041, + 3.83730050032849, 3.81602439672981, 3.79952375129251, 3.81216850705743, + 3.83475040424791, 3.84515172514695, 3.85524175050246, 3.8589833450892, + 3.86098785445721, 3.85897089799313, 3.86084133506967, 3.85946975295874, + 3.84228359847614, 3.82457407647112, 3.80871339890026, 3.8204152001564, + 3.83999679965052, 3.84951355633341, 3.04241174664212, 3.08082661685684, + 3.03341255683306, 2.91050123934182, 2.8647625390975, 2.92069124230874, + 3.20616416504156, 3.25207930188615, 3.35389278857596, 3.38203646925399, + 3.34224677531644, 3.23874199205821, 3.46488608097362, 3.49924465984842, + 3.5760328554895, 3.60145794491511, 3.57210181135635, 3.49454781701208, + 3.57210181127168, 3.60145794473248, 3.57603285522568, 3.49924465945314, + 3.46488608071886, 3.49454781688324, 3.25207930071066, 3.20616416394185, + 3.23874199136287, 3.3422467747452, 3.38203646853347, 3.35389278760403, + 2.86476253812043, 2.91050123895668, 3.0334125564585, 3.08082661625989, + 3.04241174551892, 2.92069124115386, 2.28022861731358, 2.40265429031028, + 2.4295968968027, 2.34121443274438, 2.21188698622572, 2.17953698224677, + 1.98494061622916, 1.88375470809255, 1.75116086502996, 1.70415171757556, + 1.81775136502913, 1.94602814066238, 1.5467237737173, 1.41889855692284, + 1.41889855743697, 1.54672377126063, 1.66983706264066, 1.66983706215624, + 1.81775136343869, 1.70415171324895, 1.75116086179774, 1.88375470218214, + 1.98494061271454, 1.94602813621699, 2.21188698258816, 2.34121442685979, + 2.42959689318918, 2.40265428619257, 2.28022861555447, 2.17953697788015, + 2.48250525726596, 2.58069899496666, 2.580698994828, 2.48250525939677, + 2.37040347257651, 2.37040347282208, 3.83730049953695, 3.81602439552504, + 3.79952374979441, 3.8121685055086, 3.83475040322909, 3.84515172438773, + 3.85524175001268, 3.85898334474675, 3.86098785451683, 3.85897089828832, + 3.86084133515291, 3.85946975270758, 3.8422835992181, 3.82457407735823, + 3.80871339993605, 3.82041520095658, 3.8399968002872, 3.8495135568304, + 3.83730049748391, 3.81602439274038, 3.79952374653887, 3.81216850266742, + 3.83475040109803, 3.84515172273185, 3.85524174900921, 3.85898334416065, + 3.86098785463868, 3.85897089888909, 3.86084133535606, 3.85946975218227, + 3.84228360104085, 3.82457407991773, 3.80871340299439, 3.82041520364015, + 3.83999680222929, 3.84951355826986, 3.04241173024201, 3.08082660099625, + 3.03341254061537, 2.91050122232067, 2.86476252123369, 2.92069122521197, + 3.35389277617434, 3.38203645687034, 3.34224676258644, 3.23874197779162, + 3.2061641505101, 3.25207928805274, 3.46488607017294, 3.49924464917407, + 3.57603284670063, 3.60145793609385, 3.57210180252023, 3.49454780628909, + 3.57210180267431, 3.60145793638611, 3.57603284716961, 3.4992446496989, + 3.46488607056531, 3.49454780646093, 3.38203645769644, 3.3538927774706, + 3.25207928927346, 3.206164151828, 3.23874197850084, 3.34224676335215, + 2.86476252219831, 2.91050122267359, 3.0334125410174, 3.08082660156323, + 3.04241173145599, 2.92069122628746, 2.40265427129883, 2.2802285995051, + 2.1795369623343, 2.21188696735762, 2.34121441254738, 2.42959687854648, + 2.58069897922181, 2.48250524207867, 2.37040345611661, 2.37040345608139, + 2.48250524198015, 2.58069897921053, 2.28022859968375, 2.40265427148541, + 2.42959687876723, 2.34121441266495, 2.21188696754455, 2.17953696256324, + 1.98494059721598, 1.88375468713568, 1.75116084605762, 1.70415169718831, + 1.81775134660656, 1.94602812044671, 1.41889853979862, 1.54672375422312, + 1.66983704465869, 1.6698370444596, 1.54672375443116, 1.41889853943904, + 1.88375468720816, 1.98494059721608, 1.94602812022712, 1.81775134667405, + 1.70415169707678, 1.75116084642747, 3.08082660099623, 3.04241173024201, + 2.92069122521196, 2.8647625212337, 2.91050122232066, 3.03341254061537, + 2.91050122267358, 2.86476252219832, 2.92069122628745, 3.042411731456, + 3.08082660156322, 3.0334125410174, 3.3538927774706, 3.38203645769642, + 3.34224676335215, 3.23874197850083, 3.206164151828, 3.25207928927345, + 3.46488607056531, 3.49924464969888, 3.57603284716962, 3.6014579363861, + 3.57210180267431, 3.49454780646091, 3.57210180252023, 3.60145793609383, + 3.57603284670063, 3.49924464917406, 3.46488607017293, 3.49454780628908, + 3.38203645687032, 3.35389277617434, 3.25207928805273, 3.2061641505101, + 3.2387419777916, 3.34224676258644, 3.82457407735823, 3.8422835992181, + 3.8495135568304, 3.8399968002872, 3.82041520095658, 3.80871339993605, + 3.86084133515291, 3.85897089828832, 3.86098785451683, 3.85898334474675, + 3.85524175001267, 3.85946975270758, 3.83475040322909, 3.8121685055086, + 3.79952374979441, 3.81602439552503, 3.83730049953695, 3.84515172438773, + 3.82457407991773, 3.84228360104086, 3.84951355826986, 3.8399968022293, + 3.82041520364015, 3.80871340299439, 3.85524174900921, 3.85946975218227, + 3.86084133535606, 3.85897089888909, 3.86098785463868, 3.85898334416065, + 3.83475040109803, 3.81216850266742, 3.79952374653887, 3.81602439274038, + 3.83730049748391, 3.84515172273185, 1.81775135925393, 1.7041517120716, + 1.75116086138526, 1.88375470439201, 1.98494061211483, 1.94602813482244, + 2.21188698191547, 2.34121442840182, 2.42959689251846, 2.40265428451122, + 2.28022861154444, 2.17953697632403, 2.58069898701232, 2.48250524929822, + 2.37040346431457, 2.37040346556609, 2.48250525238311, 2.58069898831853, + 2.28022860569234, 2.40265427689689, 2.42959688285938, 2.34121441649984, + 2.21188697164619, 2.17953696790764, 1.98494060130121, 1.88375469050176, + 1.75116085007805, 1.70415170224913, 1.81775135280091, 1.94602812567358, + 1.5467237617074, 1.41889854829708, 1.41889854955954, 1.54672376632688, + 1.66983705456452, 1.66983705336537, 3.46488608071885, 3.49924465945314, + 3.57603285522568, 3.60145794473248, 3.57210181127168, 3.49454781688323, + 3.57210181135634, 3.60145794491511, 3.57603285548949, 3.49924465984842, + 3.46488608097362, 3.49454781701208, 3.25207930188615, 3.20616416504156, + 3.23874199205821, 3.34224677531644, 3.38203646925399, 3.35389278857596, + 3.08082661685684, 3.04241174664212, 2.92069124230873, 2.86476253909749, + 2.91050123934182, 3.03341255683306, 2.91050123895668, 2.86476253812042, + 2.92069124115385, 3.04241174551892, 3.08082661625989, 3.0334125564585, + 3.20616416394184, 3.25207930071066, 3.35389278760403, 3.38203646853347, + 3.34224677474519, 3.23874199136287, 3.17505517819785, 3.16097552492228, + 3.05899736283131, 2.99386884005565, 3.00977836813594, 3.11994727779011, + 2.97060165769012, 2.90894311915608, 2.94084144324995, 3.06015736421514, + 3.1132519171817, 3.08618979269084, 3.58371794822061, 3.61624399712336, + 3.59477675154682, 3.52186831530198, 3.48385686711648, 3.50846685621332, + 3.60854784965156, 3.64295339381216, 3.62386338594055, 3.55882608789277, + 3.51867229952132, 3.53891420750262, 3.68589812473734, 3.70431753793426, + 3.74976895373651, 3.77230076655113, 3.75703001910139, 3.71440072749805, + 3.85215761997655, 3.84461773387907, 3.82904399367736, 3.81445584391205, + 3.82756485329104, 3.8438449880284, 3.79191280344149, 3.77128289386491, + 3.74057496637648, 3.71478050850439, 3.74057496659819, 3.77128289404559, + 3.82904399357776, 3.84461773381541, 3.85215761988125, 3.84384498783089, + 3.8275648529848, 3.81445584367382, 3.74976895195735, 3.70431753581031, + 3.68589812270217, 3.714400725886, 3.75703001776383, 3.77230076515159, + 3.60854784747611, 3.53891420513236, 3.51867229735203, 3.55882608622736, + 3.62386338439428, 3.64295339210458, 3.59477674871981, 3.61624399419415, + 3.58371794474004, 3.50846685243702, 3.48385686344634, 3.52186831222077, + 2.94084143817571, 2.90894311438688, 2.97060165380045, 3.08618978857682, + 3.11325191276977, 3.06015735901039, 3.00977836548108, 2.99386883786249, + 3.05899736141099, 3.1609755230836, 3.17505517590482, 3.11994727480999, + 2.82990733805891, 2.75803570313654, 2.63612385724924, 2.61366767714787, + 2.69774163112156, 2.81113256117699, 2.75803570716211, 2.829907341092, + 2.811132563732, 2.69774163294624, 2.61366768007544, 2.63612386069404, + 3.05899735743508, 3.16097551930725, 3.17505517241248, 3.11994727121531, + 3.00977836163036, 2.99386883382839, 2.82990733386299, 2.75803569910261, + 2.63612385273211, 2.61366767259434, 2.69774162636306, 2.81113255688698, + 2.63612385473025, 2.75803570117254, 2.8299073356907, 2.81113255834232, + 2.69774162755968, 2.61366767405634, 3.0097783632334, 3.11994727309666, + 3.17505517374158, 3.1609755206402, 3.05899735828153, 2.99386883525182, + 2.97060165232267, 2.90894311323227, 2.94084143729868, 3.06015735837716, + 3.11325191189141, 3.08618978747851, 3.58371794419843, 3.61624399366165, + 3.59477674782223, 3.52186831151937, 3.48385686281036, 3.50846685204722, + 3.60854784613328, 3.64295339082796, 3.62386338255703, 3.55882608463346, + 3.51867229580091, 3.53891420401672, 3.68589812153703, 3.70431753511233, + 3.74976895097958, 3.77230076432702, 3.75703001641713, 3.71440072485043, + 3.8521576191201, 3.84461773254023, 3.82904399225209, 3.81445584187369, + 3.82756485180208, 3.84384498665062, 3.79191280163363, 3.77128289151291, + 3.74057496432901, 3.71478050575604, 3.74057496440724, 3.77128289155391, + 3.82904399221832, 3.84461773252718, 3.85215761908784, 3.84384498660469, + 3.82756485169123, 3.8144558418203, 3.74976895027194, 3.70431753411461, + 3.68589812065336, 3.71440072405832, 3.75703001590384, 3.77230076368046, + 3.60854784508713, 3.53891420273524, 3.51867229462836, 3.55882608373335, + 3.62386338186617, 3.64295338994111, 3.59477674652184, 3.61624399231187, + 3.58371794277901, 3.5084668503841, 3.48385686111259, 3.52186830997292, + 2.90894311085056, 2.97060165015118, 3.08618978522416, 3.11325190964041, + 3.06015735595416, 2.94084143479796, 0.829324749822268, 0.719686353737887, + 0.829324750169392, 0.957414201276656, 1.04507987304047, + 0.957414200969615, 1.27627149123162, 1.40803971968493, 1.51358653750022, + 1.4563662285434, 1.32613276581893, 1.20587112141495, 1.5135865384577, + 1.40803972076972, 1.27627149303911, 1.20587112207766, 1.32613276662738, + 1.45636622869928, 1.04507987600702, 0.957414204893307, 0.829324754594157, + 0.719686356648459, 0.829324752096637, 0.957414202494864, + 1.27627149478415, 1.40803972236866, 1.51358654136619, 1.45636623337298, + 1.3261327717817, 1.20587112590036, 1.51358655314004, 1.40803973787705, + 1.27627150701867, 1.20587113710113, 1.3261327789213, 1.45636624385303, + 1.04507988867353, 0.957414219263204, 0.829324764975279, + 0.719686369895612, 0.829324762817598, 0.957414216989921, 1.2762715086727, + 1.40803973958239, 1.51358655598572, 1.45636624871029, 1.3261327836562, + 1.20587114092256, 1.51358655319684, 1.40803973480565, 1.27627150647983, + 1.20587113747749, 1.32613278250847, 1.4563662447156, 1.04507988721126, + 0.957414214549234, 0.829324762185518, 0.719686368197244, 0.8293247645036, + 0.957414216583879, 1.27627150457475, 1.40803973320339, 1.51358654975237, + 1.45636624000749, 1.32613277690619, 1.20587113394636, 1.51358654290189, + 1.40803972445672, 1.27627149654387, 1.20587112747997, 1.3261327725023, + 1.45636623451286, 1.04507987780354, 0.957414204615115, 0.8293247537153, + 0.71968635851059, 0.829324755789629, 0.957414206727861, 1.27627149516361, + 1.4080397231698, 1.5135865406883, 1.45636623078723, 1.3261327684941, + 1.20587112426236, 1.40803971976829, 1.27627149136335, 1.20587112180345, + 1.32613276625679, 1.45636622883327, 1.51358653755931, 3.17505517590482, + 3.16097552308359, 3.05899736141099, 2.99386883786248, 3.00977836548107, + 3.11994727480998, 2.97060165380045, 2.90894311438687, 2.94084143817571, + 3.06015735901038, 3.11325191276977, 3.08618978857681, 3.58371794474004, + 3.61624399419416, 3.59477674871981, 3.52186831222077, 3.48385686344634, + 3.50846685243703, 3.6085478474761, 3.64295339210458, 3.62386338439427, + 3.55882608622737, 3.51867229735202, 3.53891420513237, 3.68589812270216, + 3.70431753581031, 3.74976895195734, 3.77230076515159, 3.75703001776383, + 3.714400725886, 3.85215761988125, 3.8446177338154, 3.82904399357776, + 3.81445584367381, 3.82756485298481, 3.84384498783089, 3.7919128034415, + 3.77128289404559, 3.74057496659821, 3.71478050850439, 3.74057496637649, + 3.77128289386491, 3.82904399367736, 3.84461773387906, 3.85215761997655, + 3.8438449880284, 3.82756485329105, 3.81445584391204, 3.74976895373651, + 3.70431753793427, 3.68589812473734, 3.71440072749807, 3.75703001910139, + 3.77230076655114, 3.60854784965156, 3.53891420750263, 3.51867229952133, + 3.55882608789279, 3.62386338594055, 3.64295339381217, 3.59477675154682, + 3.61624399712337, 3.58371794822061, 3.50846685621333, 3.48385686711648, + 3.52186831530199, 2.94084144324996, 2.90894311915608, 2.97060165769014, + 3.08618979269085, 3.11325191718171, 3.06015736421513, 3.00977836813596, + 2.99386884005566, 3.05899736283132, 3.16097552492229, 3.17505517819786, + 3.11994727779012, 2.82990734109201, 2.75803570716211, 2.63612386069404, + 2.61366768007543, 2.69774163294624, 2.81113256373201, 2.75803570313653, + 2.82990733805891, 2.81113256117699, 2.69774163112156, 2.61366767714787, + 2.63612385724924, 3.74057496440724, 3.71478050575604, 3.74057496432901, + 3.77128289151291, 3.79191280163363, 3.7712828915539, 3.82904399225209, + 3.84461773254023, 3.8521576191201, 3.84384498665062, 3.82756485180208, + 3.81445584187369, 3.74976895097957, 3.70431753511233, 3.68589812153703, + 3.71440072485042, 3.75703001641713, 3.77230076432701, 3.60854784613328, + 3.53891420401671, 3.5186722958009, 3.55882608463346, 3.62386338255703, + 3.64295339082796, 3.59477674782222, 3.61624399366165, 3.58371794419843, + 3.50846685204721, 3.48385686281036, 3.52186831151937, 2.94084143729867, + 2.90894311323226, 2.97060165232266, 3.0861897874785, 3.11325191189141, + 3.06015735837715, 3.00977836323339, 2.99386883525181, 3.05899735828152, + 3.1609755206402, 3.17505517374158, 3.11994727309664, 2.82990733569069, + 2.75803570117252, 2.63612385473025, 2.61366767405633, 2.69774162755968, + 2.8111325583423, 2.6361238527321, 2.75803569910262, 2.829907333863, + 2.811132556887, 2.69774162636306, 2.61366767259434, 3.00977836163036, + 3.11994727121532, 3.17505517241248, 3.16097551930725, 3.05899735743508, + 2.99386883382839, 2.97060165015119, 2.90894311085058, 2.94084143479797, + 3.06015735595416, 3.11325190964042, 3.08618978522417, 3.583717942779, + 3.61624399231187, 3.59477674652184, 3.52186830997292, 3.4838568611126, + 3.5084668503841, 3.60854784508713, 3.64295338994111, 3.62386338186617, + 3.55882608373334, 3.51867229462835, 3.53891420273523, 3.68589812065336, + 3.7043175341146, 3.74976895027195, 3.77230076368046, 3.75703001590384, + 3.71440072405832, 3.84461773252718, 3.82904399221832, 3.8144558418203, + 3.82756485169123, 3.84384498660469, 3.85215761908784, 3.51867230681858, + 3.55882609482331, 3.62386339196214, 3.64295339969012, 3.60854785582946, + 3.53891421460591, 3.59477675781379, 3.61624400319028, 3.58371795445638, + 3.50846686334877, 3.48385687456329, 3.52186832248982, 2.94084145333509, + 2.90894313019809, 2.97060166803225, 3.08618980314812, 3.11325192691397, + 3.06015737450286, 3.00977837854318, 2.99386885102277, 3.05899737318336, + 3.16097553510326, 3.17505518797796, 3.11994728818731, 2.82990735187877, + 2.75803571871458, 2.63612387157793, 2.61366769189825, 2.69774164402701, + 2.81113257525686, 2.63612386811255, 2.75803571373755, 2.82990734865515, + 2.81113257199718, 2.69774164234379, 2.6136676881632, 3.00977837562289, + 3.11994728458699, 3.17505518540039, 3.16097553290413, 3.05899737171443, + 2.99386884828909, 2.97060166362898, 2.90894312436941, 2.94084144754962, + 3.06015736820745, 3.11325192172779, 3.08618979814766, 3.58371795056125, + 3.61624399975918, 3.59477675467844, 3.52186831884396, 3.4838568703685, + 3.50846685883469, 3.60854785345252, 3.6429533977543, 3.62386339029672, + 3.55882609292684, 3.51867230442272, 3.53891421185193, 3.68589812755707, + 3.7043175403825, 3.74976895570274, 3.77230076861619, 3.75703002144123, + 3.71440073044731, 3.85215762093028, 3.84461773513492, 3.82904399568355, + 3.81445584604517, 3.82756485512302, 3.84384498917669, 3.79191280667193, + 3.77128289754252, 3.74057497097852, 3.71478051314172, 3.74057497073431, + 3.77128289734671, 3.82904399579575, 3.84461773521061, 3.85215762104181, + 3.84384498940757, 3.82756485546706, 3.81445584631225, 3.70431754280801, + 3.68589812975883, 3.71440073226354, 3.75703002287439, 3.77230077018087, + 3.7497689576034, 3.6238633838419, 3.55882608566565, 3.51867229680122, + 3.53891420454323, 3.60854784690197, 3.64295339161911, 3.68589812208833, + 3.70431753530002, 3.74976895133279, 3.77230076470937, 3.75703001715803, + 3.7144007254256, 3.85215761968578, 3.84461773349495, 3.82904399331002, + 3.8144558432207, 3.82756485267547, 3.84384498745929, 3.79191280314032, + 3.7712828936652, 3.74057496622281, 3.71478050808158, 3.74057496601206, + 3.77128289341573, 3.82904399344128, 3.84461773361157, 3.85215761983193, + 3.84384498780549, 3.82756485306006, 3.81445584359317, 3.74976895293871, + 3.70431753660344, 3.68589812369588, 3.71440072652904, 3.75703001853086, + 3.77230076581546, 3.60854784835546, 3.53891420554641, 3.51867229795721, + 3.55882608649137, 3.62386338504842, 3.64295339261555, 3.59477674977981, + 3.61624399496551, 3.58371794595683, 3.50846685302161, 3.48385686437887, + 3.52186831273195, 2.94084143674032, 2.90894311305048, 2.97060165237489, + 3.08618978786781, 3.11325191197401, 3.06015735845159, 3.0097783640784, + 2.99386883625511, 3.05899735997717, 3.16097552205816, 3.17505517514038, + 3.1199472739597, 2.82990733591079, 2.75803570097618, 2.63612385431683, + 2.61366767409011, 2.69774162810981, 2.81113255888408, 2.63612385444689, + 2.75803570096113, 2.82990733585776, 2.81113255894476, 2.69774162815067, + 2.61366767421521, 3.00977836378975, 3.11994727369897, 3.17505517461689, + 3.16097552195971, 3.05899735965723, 2.99386883624721, 2.97060165209063, + 2.90894311271201, 2.94084143634347, 3.06015735763346, 3.11325191135367, + 3.08618978738398, 3.61624399356675, 3.59477674808602, 3.52186831154629, + 3.48385686272553, 3.50846685160897, 3.5837179439899, 3.51867229680123, + 3.55882608566566, 3.62386338384191, 3.64295339161911, 3.60854784690197, + 3.53891420454323, 3.59477674808602, 3.61624399356676, 3.58371794398991, + 3.50846685160897, 3.48385686272553, 3.52186831154629, 2.94084143634348, + 2.90894311271201, 2.97060165209063, 3.08618978738399, 3.11325191135367, + 3.06015735763348, 3.00977836378975, 2.99386883624721, 3.05899735965724, + 3.16097552195972, 3.17505517461689, 3.11994727369899, 2.82990733585777, + 2.75803570096114, 2.63612385444689, 2.61366767421522, 2.69774162815068, + 2.81113255894476, 2.63612385431684, 2.75803570097618, 2.82990733591078, + 2.81113255888409, 2.69774162810982, 2.61366767409012, 3.0097783640784, + 3.1199472739597, 3.17505517514038, 3.16097552205816, 3.05899735997718, + 2.99386883625512, 2.97060165237488, 2.90894311305048, 2.94084143674031, + 3.06015735845159, 3.113251911974, 3.08618978786781, 3.58371794595682, + 3.61624399496549, 3.59477674977981, 3.52186831273193, 3.48385686437887, + 3.5084668530216, 3.60854784835546, 3.64295339261554, 3.62386338504842, + 3.55882608649137, 3.51867229795721, 3.53891420554639, 3.68589812369588, + 3.70431753660344, 3.74976895293871, 3.77230076581545, 3.75703001853087, + 3.71440072652904, 3.85215761983192, 3.84461773361157, 3.82904399344129, + 3.81445584359317, 3.82756485306006, 3.84384498780549, 3.79191280314032, + 3.77128289341573, 3.74057496601206, 3.71478050808159, 3.74057496622281, + 3.7712828936652, 3.82904399331002, 3.84461773349495, 3.85215761968578, + 3.84384498745929, 3.82756485267547, 3.8144558432207, 3.70431753530002, + 3.68589812208834, 3.7144007254256, 3.75703001715803, 3.77230076470937, + 3.74976895133279, 0.829324750169394, 0.71968635373788, 0.829324749822264, + 0.957414200969637, 1.04507987304049, 0.957414201276675, 1.27627149136335, + 1.4080397197683, 1.51358653755931, 1.45636622883328, 1.3261327662568, + 1.20587112180346, 1.5135865406883, 1.40803972316978, 1.27627149516361, + 1.20587112426234, 1.3261327684941, 1.45636623078722, 1.04507987780355, + 0.957414206727851, 0.829324755789636, 0.719686358510584, + 0.829324753715309, 0.957414204615115, 1.27627149654387, 1.40803972445672, + 1.5135865429019, 1.45636623451286, 1.32613277250231, 1.20587112747998, + 1.51358654975237, 1.40803973320338, 1.27627150457475, 1.20587113394636, + 1.32613277690619, 1.45636624000748, 1.04507988721126, 0.957414216583866, + 0.829324764503594, 0.719686368197235, 0.829324762185524, + 0.957414214549229, 1.27627150647983, 1.40803973480564, 1.51358655319684, + 1.45636624471558, 1.32613278250847, 1.20587113747747, 1.51358655598572, + 1.40803973958239, 1.2762715086727, 1.20587114092257, 1.32613278365619, + 1.45636624871029, 1.04507988867352, 0.957414216989908, 0.829324762817598, + 0.719686369895601, 0.829324764975269, 0.957414219263202, + 1.27627150701867, 1.40803973787704, 1.51358655314004, 1.45636624385303, + 1.32613277892131, 1.20587113710114, 1.51358654136616, 1.40803972236866, + 1.27627149478413, 1.20587112590037, 1.3261327717817, 1.45636623337298, + 1.045079876007, 0.957414202494867, 0.829324752096628, 0.719686356648451, + 0.829324754594135, 0.957414204893297, 1.27627149303908, 1.40803972076972, + 1.51358653845769, 1.45636622869928, 1.32613276662738, 1.20587112207765, + 1.40803971968494, 1.27627149123163, 1.20587112141496, 1.32613276581896, + 1.4563662285434, 1.51358653750022, 3.51867230530465, 3.55882609310292, + 3.62386339067989, 3.64295339853009, 3.60854785484376, 3.53891421325143, + 3.59477675710285, 3.61624400257861, 3.58371795398677, 3.50846686266358, + 3.48385687373568, 3.52186832148399, 2.94084145141462, 2.90894312732319, + 2.97060166536041, 3.0861898005471, 3.11325192510552, 3.06015737263463, + 3.00977837515217, 2.99386884646424, 3.05899736912919, 3.16097553114405, + 3.1750551849751, 3.11994728488167, 2.82990734810756, 2.75803571461202, + 2.63612386749075, 2.61366768638992, 2.69774163895515, 2.81113257017222, + 2.63612385851353, 2.75803570508642, 2.82990733988544, 2.81113256429108, + 2.69774163377649, 2.61366767963183, 3.00977836776533, 3.11994727781453, + 3.17505517852964, 3.16097552690989, 3.05899736478825, 2.99386884143827, + 2.97060165508268, 2.90894311601725, 2.94084143858907, 3.06015736050123, + 3.11325191378591, 3.08618979079264, 3.58371794676065, 3.61624399590161, + 3.59477675095128, 3.5218683140611, 3.48385686560797, 3.50846685392672, + 3.60854784994205, 3.64295339436361, 3.62386338712135, 3.55882608885917, + 3.5186723001745, 3.53891420735824, 3.68589812482293, 3.70431753752687, + 3.74976895372371, 3.77230076667804, 3.7570300196041, 3.71440072778249, + 3.85215762056045, 3.84461773471831, 3.82904399494469, 3.81445584521316, + 3.82756485425672, 3.84384498871125, 3.79191280555792, 3.771282896499, + 3.74057496950739, 3.71478051155035, 3.7405749689668, 3.77128289612978, + 3.82904399516701, 3.84461773483012, 3.85215762076234, 3.84384498906083, + 3.82756485493774, 3.81445584566869, 3.70431754228365, 3.68589812913556, + 3.71440073152898, 3.75703002232892, 3.77230076974821, 3.74976895722169, + 3.62386339067989, 3.55882609310293, 3.51867230530466, 3.53891421325144, + 3.60854785484375, 3.64295339853008, 3.68589812913556, 3.70431754228365, + 3.74976895722168, 3.77230076974821, 3.75703002232892, 3.71440073152898, + 3.85215762076234, 3.84461773483012, 3.82904399516701, 3.8144558456687, + 3.82756485493774, 3.84384498906083, 3.79191280555792, 3.77128289612979, + 3.7405749689668, 3.71478051155035, 3.74057496950739, 3.77128289649901, + 3.8290439949447, 3.84461773471831, 3.85215762056045, 3.84384498871125, + 3.82756485425673, 3.81445584521317, 3.74976895372371, 3.70431753752687, + 3.68589812482294, 3.71440072778249, 3.7570300196041, 3.77230076667804, + 3.60854784994206, 3.53891420735824, 3.51867230017451, 3.55882608885918, + 3.62386338712136, 3.64295339436362, 3.59477675095128, 3.61624399590161, + 3.58371794676065, 3.50846685392671, 3.48385686560797, 3.5218683140611, + 2.94084143858907, 2.90894311601726, 2.97060165508268, 3.08618979079265, + 3.11325191378591, 3.06015736050124, 3.00977836776534, 2.99386884143828, + 3.05899736478827, 3.1609755269099, 3.17505517852965, 3.11994727781455, + 2.82990733988545, 2.75803570508643, 2.63612385851352, 2.61366767963184, + 2.69774163377649, 2.8111325642911, 2.63612386749076, 2.75803571461203, + 2.82990734810755, 2.81113257017222, 2.69774163895516, 2.61366768638993, + 3.00977837515218, 3.11994728488168, 3.1750551849751, 3.16097553114405, + 3.0589973691292, 2.99386884646425, 2.97060166536041, 2.90894312732319, + 2.94084145141462, 3.06015737263463, 3.11325192510551, 3.0861898005471, + 3.61624400257861, 3.59477675710285, 3.52186832148399, 3.48385687373568, + 3.50846686266358, 3.58371795398677, 3.62386339196214, 3.55882609482331, + 3.51867230681857, 3.5389142146059, 3.60854785582946, 3.64295339969012, + 3.68589812975883, 3.70431754280801, 3.7497689576034, 3.77230077018087, + 3.75703002287439, 3.71440073226354, 3.85215762104181, 3.84461773521062, + 3.82904399579574, 3.81445584631225, 3.82756485546705, 3.84384498940757, + 3.79191280667193, 3.77128289734671, 3.74057497073431, 3.71478051314172, + 3.74057497097852, 3.77128289754252, 3.82904399568355, 3.84461773513492, + 3.85215762093028, 3.84384498917669, 3.82756485512302, 3.81445584604517, + 3.74976895570274, 3.7043175403825, 3.68589812755708, 3.71440073044731, + 3.75703002144123, 3.77230076861619, 3.60854785345253, 3.53891421185193, + 3.51867230442274, 3.55882609292684, 3.62386339029672, 3.6429533977543, + 3.59477675467844, 3.61624399975918, 3.58371795056126, 3.50846685883469, + 3.48385687036851, 3.52186831884397, 2.94084144754963, 2.90894312436943, + 2.97060166362898, 3.08618979814767, 3.1132519217278, 3.06015736820747, + 3.0097783756229, 2.9938688482891, 3.05899737171444, 3.16097553290414, + 3.17505518540039, 3.11994728458699, 2.82990734865515, 2.75803571373757, + 2.63612386811256, 2.61366768816322, 2.6977416423438, 2.8111325719972, + 2.63612387157793, 2.75803571871459, 2.82990735187878, 2.81113257525687, + 2.69774164402702, 2.61366769189824, 3.00977837854319, 3.11994728818731, + 3.17505518797796, 3.16097553510325, 3.05899737318337, 2.99386885102277, + 2.97060166803226, 2.90894313019808, 2.94084145333509, 3.06015737450286, + 3.11325192691397, 3.08618980314813, 3.61624400319029, 3.59477675781379, + 3.52186832248982, 3.48385687456328, 3.50846686334877, 3.58371795445638, + 3.80788075066933, 3.793359430218, 3.80935514961242, 3.83281418047548, + 3.84205468437419, 3.83178045932269, 3.75630308120772, 3.73671407998881, + 3.6873827382772, 3.66288489121358, 3.68738273866529, 3.73671408030886, + 3.83178045917173, 3.84205468411383, 3.83281418003925, 3.8093551489737, + 3.79335942975339, 3.80788075045042, 2.27781169371947, 2.35355625792149, + 2.29919933349799, 2.16556407711254, 2.07741510386896, 2.14206673667462, + 2.47890799981366, 2.5433708395083, 2.67586026364294, 2.7303060630097, + 2.67586026267281, 2.54337083843576, 2.35355626028493, 2.27781169484832, + 2.14206673797471, 2.07741510630394, 2.16556408100954, 2.29919933687183, + 3.73671407846663, 3.75630307940097, 3.73671407836399, 3.68738273582522, + 3.66288488872016, 3.6873827359458, 3.79335942881371, 3.80788074939396, + 3.83178045860985, 3.84205468362, 3.83281417958652, 3.80935514809748, + 3.83178045866981, 3.80788074946883, 3.793359428979, 3.80935514832405, + 3.8328141797671, 3.84205468371904, 3.80788074832992, 3.83178045753363, + 3.84205468286234, 3.83281417846865, 3.80935514701321, 3.79335942734603, + 3.83178045763168, 3.80788074840196, 3.79335942757774, 3.80935514723929, + 3.83281417876637, 3.84205468298099, 3.73671407681723, 3.75630307816502, + 3.73671407670314, 3.68738273458223, 3.66288488699152, 3.68738273464747, + 2.27781168468752, 2.35355624971977, 2.29919932524649, 2.16556406900792, + 2.07741509487063, 2.14206672772056, 2.54337083020097, 2.47890799077767, + 2.54337083048016, 2.67586025464235, 2.73030605480022, 2.67586025443279, + 2.35355625050973, 2.2778116851111, 2.14206672824714, 2.07741509602769, + 2.16556407062235, 2.29919932655198, 3.75630308447383, 3.73671408351201, + 3.68738274269091, 3.66288489561696, 3.68738274275124, 3.73671408355243, + 3.79335943233127, 3.80788075280813, 3.83178046073998, 3.84205468551873, + 3.8328141816563, 3.8093551514171, 3.83178046075959, 3.80788075283164, + 3.79335943239049, 3.80935515149345, 3.83281418171944, 3.84205468554513, + 3.80788074787498, 3.83178045717735, 3.84205468265246, 3.83281417831282, + 3.80935514676775, 3.79335942695681, 3.83178045708815, 3.80788074775654, + 3.79335942669308, 3.80935514642365, 3.83281417805311, 3.84205468250755, + 3.73671407601969, 3.68738273376602, 3.66288488597554, 3.68738273356469, + 3.73671407584276, 3.75630307741336, 2.27781168194595, 2.35355624725223, + 2.29919932326004, 2.16556406678664, 2.07741509233704, 2.14206672464747, + 2.54337082758285, 2.47890798793938, 2.54337082725197, 2.6758602517937, + 2.7303060521297, 2.67586025212544, 2.35355624667329, 2.27781168170246, + 2.14206672440549, 2.07741509198883, 2.16556406611879, 2.29919932257358, + 2.27781168284262, 2.14206672539865, 2.07741509274297, 2.16556406687647, + 2.29919932348106, 2.35355624761155, 2.14206672623215, 2.27781168362573, + 2.3535562489834, 2.29919932583701, 2.16556406939985, 2.07741509459602, + 2.54337082892105, 2.47890798945701, 2.54337082836692, 2.67586025344429, + 2.73030605344056, 2.67586025401124, 2.54337084690776, 2.47890800846272, + 2.5433708462933, 2.67586027161076, 2.73030607102055, 2.67586027222717, + 2.35355626523264, 2.27781170199606, 2.14206674320176, 2.07741511130966, + 2.16556408348771, 2.29919934142396, 2.35355626673157, 2.29919934411479, + 2.16556408615858, 2.07741511345627, 2.14206674410725, 2.27781170290775, + 3.75630307940099, 3.73671407846663, 3.68738273594581, 3.66288488872017, + 3.68738273582524, 3.73671407836399, 3.80788074946884, 3.83178045866981, + 3.84205468371904, 3.8328141797671, 3.80935514832406, 3.793359428979, + 3.79335942881371, 3.80935514809749, 3.83281417958652, 3.84205468362001, + 3.83178045860986, 3.80788074939397, 3.75630307816503, 3.73671407681723, + 3.68738273464747, 3.66288488699152, 3.68738273458223, 3.73671407670314, + 3.80788074840196, 3.83178045763168, 3.84205468298099, 3.83281417876637, + 3.80935514723929, 3.79335942757774, 3.83178045753363, 3.80788074832992, + 3.79335942734603, 3.80935514701322, 3.83281417846865, 3.84205468286235, + 3.793359430218, 3.80788075066933, 3.83178045932269, 3.84205468437419, + 3.83281418047548, 3.80935514961242, 3.83178045917173, 3.80788075045042, + 3.79335942975339, 3.8093551489737, 3.83281418003925, 3.84205468411383, + 3.7367140799888, 3.75630308120773, 3.73671408030886, 3.68738273866529, + 3.66288489121358, 3.6873827382772, 2.27781169484829, 2.35355626028491, + 2.29919933687181, 2.16556408100954, 2.07741510630392, 2.14206673797472, + 2.54337083950829, 2.47890799981363, 2.54337083843575, 2.6758602626728, + 2.7303060630097, 2.67586026364293, 2.35355625792149, 2.27781169371944, + 2.14206673667463, 2.07741510386894, 2.16556407711255, 2.29919933349798, + 2.35355624971976, 2.27781168468751, 2.14206672772055, 2.07741509487061, + 2.16556406900791, 2.29919932524647, 2.27781168511109, 2.35355625050974, + 2.29919932655197, 2.16556407062236, 2.07741509602769, 2.14206672824714, + 2.54337083020097, 2.67586025443278, 2.73030605480022, 2.67586025464233, + 2.54337083048016, 2.47890799077766, 3.80788075283164, 3.83178046075959, + 3.84205468554513, 3.83281418171944, 3.80935515149345, 3.79335943239049, + 3.80788075280813, 3.79335943233126, 3.8093551514171, 3.8328141816563, + 3.84205468551873, 3.83178046073998, 3.73671408351201, 3.75630308447383, + 3.73671408355242, 3.68738274275125, 3.66288489561696, 3.6873827426909, + 2.35355624725224, 2.27781168194597, 2.14206672464747, 2.07741509233704, + 2.16556406678664, 2.29919932326005, 2.27781168170248, 2.3535562466733, + 2.2991993225736, 2.16556406611881, 2.07741509198885, 2.1420667244055, + 2.54337082758287, 2.67586025212545, 2.73030605212971, 2.67586025179371, + 2.54337082725199, 2.47890798793941, 3.83178045717735, 3.80788074787498, + 3.79335942695681, 3.80935514676775, 3.83281417831282, 3.84205468265246, + 3.73671407601969, 3.75630307741336, 3.73671407584277, 3.68738273356468, + 3.66288488597554, 3.68738273376602, 3.83178045708815, 3.84205468250755, + 3.83281417805311, 3.80935514642365, 3.79335942669308, 3.80788074775654, + 2.27781170199606, 2.35355626523265, 2.29919934142398, 2.16556408348772, + 2.07741511130968, 2.14206674320177, 2.47890800846272, 2.54337084690776, + 2.67586027222717, 2.73030607102056, 2.67586027161076, 2.5433708462933, + 2.35355626673158, 2.27781170290776, 2.14206674410724, 2.07741511345628, + 2.16556408615858, 2.2991993441148, 2.27781168362574, 2.14206672623214, + 2.07741509459602, 2.16556406939983, 2.29919932583703, 2.3535562489834, + 2.14206672539864, 2.27781168284262, 2.35355624761154, 2.29919932348108, + 2.16556406687647, 2.07741509274298, 2.54337082892103, 2.67586025401125, + 2.73030605344055, 2.67586025344429, 2.54337082836691, 2.47890798945702, + 3.56582381262452, 3.49179372077245, 3.35098504543044, 3.35098504600089, + 3.49179372147866, 3.49179371820619, 3.5658238094764, 3.49179371789595, + 3.35098504184857, 3.35098504242207, 0.27021038919352, 0.270210381080679, + 0.270210376352625, 0.270210379799991, 0.270210389398549, + 3.49179372147868, 3.35098504600089, 3.35098504543044, 3.49179372077246, + 3.56582381262451, 3.49179371789595, 3.5658238094764, 3.49179371820619, + 3.35098504242206, 3.35098504184857, 3.49179372913151, 3.35098505489057, + 3.35098505425721, 3.49179372833768, 3.56582381947488, 3.35098504458005, + 3.49179372001333, 3.56582381207765, 3.49179372042905, 3.35098504457686, + 3.49179372042905, 3.56582381207766, 3.49179372001333, 3.35098504458005, + 3.35098504457686, 0.27021038939853, 0.270210379800009, 0.270210376352618, + 0.270210381080673, 0.270210389193507, 3.49179372435823, 3.56582381677044, + 3.49179372630058, 3.3509850510875, 3.35098504989944, 3.49179372630059, + 3.56582381677044, 3.49179372435824, 3.35098504989946, 3.3509850510875, + 3.35098505425721, 3.35098505489056, 3.49179372913152, 3.56582381947489, + 3.49179372833769, 3.19535085205332, 3.07668323326383, 3.19535085352684, + 3.37359787129475, 3.46244566403506, 3.37359787006572, 3.84736470560324, + 3.85625299887573, 3.85830508739538, 3.84736470580818, 3.85625299898753, + 3.85830508731231, 3.85830508724626, 3.85625299905455, 3.84736470599368, + 3.8583050875722, 3.85625299862475, 3.84736470530559, 3.37359786470753, + 3.19535084579305, 3.46244565827196, 3.37359786459013, 3.19535084565499, + 3.07668322603769, 2.26471331729362, 2.0537377418623, 1.79512453701271, + 1.7951245390242, 2.05373774457892, 2.264713318928, 3.84736470479973, + 3.85625299829885, 3.85830508781515, 3.84736470699574, 3.8562529996109, + 3.85830508681783, 3.85830508677252, 3.85625299968159, 3.84736470714796, + 3.85830508792246, 3.8562529981672, 3.84736470458645, 3.37359786276103, + 3.46244565686712, 3.37359786333627, 3.1953508442598, 3.07668322389714, + 3.19535084354769, 2.26471331183879, 2.0537377360699, 1.79512453087718, + 1.7951245314102, 2.05373773758979, 2.26471331239681, 2.26471332332703, + 2.05373774888787, 1.79512454228193, 1.79512453798091, 2.05373774273307, + 2.26471331979257, 1.79512454988451, 1.79512454734629, 2.05373775476819, + 2.2647133307081, 2.26471332835024, 2.05373775305884, 1.7951245306023, + 2.0537377365977, 2.26471331244287, 2.26471331230658, 2.05373773648767, + 1.79512453038448, 3.84736470530559, 3.85830508724626, 3.85625299862475, + 3.8583050875722, 3.84736470599368, 3.85625299905455, 3.85830508739538, + 3.84736470580818, 3.85625299887573, 3.84736470560324, 3.85830508731231, + 3.85625299898753, 3.19535085352682, 3.07668323326383, 3.1953508520533, + 3.37359787006572, 3.46244566403506, 3.37359787129476, 2.264713318928, + 2.05373774457893, 1.79512453902421, 1.79512453701272, 2.05373774186231, + 2.26471331729362, 3.19535084579303, 3.07668322603768, 3.19535084565498, + 3.37359786459013, 3.46244565827196, 3.37359786470753, 3.85830508681783, + 3.8562529996109, 3.84736470699574, 3.85830508781515, 3.85625299829884, + 3.84736470479973, 1.79512453087719, 2.0537377360699, 2.2647133118388, + 2.2647133123968, 2.05373773758979, 1.79512453141019, 3.37359786333627, + 3.46244565686712, 3.37359786276104, 3.1953508435477, 3.07668322389715, + 3.1953508442598, 3.84736470458645, 3.85830508677252, 3.8562529981672, + 3.85830508792246, 3.84736470714795, 3.85625299968159, 3.19535086027984, + 3.07668324119428, 3.19535086082623, 3.37359787765526, 3.46244567038539, + 3.37359787730248, 2.2647133307081, 2.05373775476817, 1.79512454988451, + 1.79512454734627, 2.05373775305883, 2.26471332835022, 3.84736470659195, + 3.85830508775007, 3.85625299931045, 3.8583050870498, 3.84736470503321, + 3.85625299838543, 3.1953508468231, 3.07668322562571, 3.19535084617094, + 3.37359786505137, 3.46244565973068, 3.37359786545367, 1.79512453038448, + 2.05373773648765, 2.26471331230659, 2.26471331244288, 2.0537377365977, + 1.7951245306023, 3.19535084617094, 3.0766832256257, 3.1953508468231, + 3.37359786545366, 3.46244565973068, 3.37359786505135, 3.85625299838543, + 3.84736470503321, 3.8583050870498, 3.85625299931045, 3.84736470659195, + 3.85830508775007, 1.79512454228193, 2.05373774888787, 2.26471332332703, + 2.26471331979257, 2.05373774273307, 1.79512453798092, 3.37359787730248, + 3.19535086027983, 3.46244567038539, 3.37359787765525, 3.19535086082623, + 3.07668324119428, 3.49962488067984, 3.43713675798983, 3.2772942422533, + 3.14070016124613, 3.22380484984836, 3.39577856947789, 3.65861887353402, + 3.69666641030478, 3.77954577861335, 3.81231167451188, 3.79051717653684, + 3.71291500127558, 3.71291500077803, 3.79051717621703, 3.81231167402121, + 3.77954577760484, 3.69666640880429, 3.65861887241131, 3.14070015788068, + 3.27729423978309, 3.43713675573172, 3.49962487776198, 3.39577856561065, + 3.22380484540448, 3.05128051322848, 2.88592335054783, 2.68698336414128, + 2.68698336541731, 2.88592335215934, 3.05128051414338, 3.0512805090478, + 2.88592334617744, 2.68698335936481, 2.6869833600004, 2.88592334721774, + 3.05128050974972, 3.49962487671691, 3.43713675423585, 3.27729423807334, + 3.14070015643117, 3.22380484474151, 3.3957785650321, 3.79051717462295, + 3.71291499836736, 3.65861887072152, 3.69666640720477, 3.77954577649199, + 3.8123116724299, 3.69666640659036, 3.65861887018596, 3.71291499823603, + 3.79051717450862, 3.81231167229886, 3.77954577603718, 3.43713675297417, + 3.49962487520551, 3.39577856316439, 3.22380484250529, 3.14070015453916, + 3.27729423642931, 0.77165547598209, 1.07623094619881, 1.29180705383508, + 1.29180705371894, 1.07623094762527, 0.771655476539705, 0.771655482204467, + 1.07623095174485, 1.29180706091197, 1.29180706584451, 1.07623095935429, + 0.771655488096443, 0.771655493061666, 1.07623096303874, 1.29180707275934, + 1.29180707050859, 1.07623096220338, 0.771655491577014, 0.77165548712907, + 1.07623095806281, 1.29180706380607, 1.2918070612426, 1.07623095277605, + 0.771655483333819, 0.771655478205974, 0.771655476779375, + 1.07623094945772, 1.29180705531781, 1.29180705463828, 1.07623094668678, + 3.27729423978309, 3.14070015788068, 3.22380484540448, 3.39577856561065, + 3.49962487776198, 3.43713675573173, 3.65861887241132, 3.69666640880429, + 3.77954577760484, 3.81231167402121, 3.79051717621703, 3.71291500077803, + 3.69666641030478, 3.65861887353403, 3.71291500127557, 3.79051717653685, + 3.81231167451188, 3.77954577861336, 3.43713675798985, 3.49962488067984, + 3.3957785694779, 3.22380484984836, 3.14070016124615, 3.2772942422533, + 3.05128051414338, 2.88592335215934, 2.68698336541731, 2.68698336414127, + 2.88592335054783, 3.05128051322848, 3.71291499836736, 3.79051717462295, + 3.8123116724299, 3.77954577649199, 3.69666640720476, 3.65861887072152, + 3.43713675423583, 3.49962487671691, 3.3957785650321, 3.22380484474149, + 3.14070015643116, 3.27729423807334, 2.88592334617744, 3.05128050904779, + 3.05128050974972, 2.88592334721774, 2.68698336000039, 2.68698335936481, + 3.49962487520551, 3.43713675297416, 3.27729423642931, 3.14070015453917, + 3.22380484250529, 3.3957785631644, 3.65861887018596, 3.69666640659035, + 3.77954577603718, 3.81231167229886, 3.79051717450862, 3.71291499823603, + 3.43713676607313, 3.49962488811077, 3.39577857766595, 3.22380485941718, + 3.14070017107195, 3.27729425170688, 3.05128052482442, 2.88592336299732, + 2.68698337712499, 2.68698337528081, 2.88592336141154, 3.05128052362768, + 3.27729424882958, 3.14070016717738, 3.22380485410074, 3.39577857302032, + 3.49962488482424, 3.43713676345293, 3.65861887797203, 3.69666641359204, + 3.77954578099049, 3.81231167644173, 3.79051717945454, 3.71291500542055, + 3.71291500595043, 3.79051717980798, 3.81231167698461, 3.77954578211128, + 3.69666641520302, 3.65861887922611, 3.79051717588132, 3.71291500028723, + 3.65861887194212, 3.69666640821601, 3.77954577718953, 3.81231167350748, + 3.69666640957692, 3.65861887271224, 3.71291500086573, 3.79051717623123, + 3.81231167422142, 3.77954577809428, 3.43713675581096, 3.49962487854547, + 3.39577856581431, 3.22380484540421, 3.14070015711114, 3.27729423946488, + 2.88592334824201, 3.05128051151309, 3.0512805114258, 2.88592334836884, + 2.68698336090578, 2.68698336098713, 3.27729423901026, 3.14070015657375, + 3.22380484442033, 3.39577856469585, 3.49962487716159, 3.43713675500253, + 3.14070015657376, 3.27729423901026, 3.43713675500254, 3.49962487716159, + 3.39577856469585, 3.22380484442034, 3.05128051151309, 2.88592334824202, + 2.68698336098714, 2.6869833609058, 2.88592334836885, 3.05128051142581, + 3.49962487854547, 3.43713675581094, 3.27729423946488, 3.14070015711113, + 3.22380484540421, 3.39577856581429, 3.65861887271223, 3.69666640957692, + 3.77954577809428, 3.81231167422143, 3.79051717623123, 3.71291500086573, + 3.71291500028723, 3.79051717588132, 3.81231167350748, 3.77954577718954, + 3.69666640821602, 3.65861887194212, 0.771655476779391, 1.07623094668678, + 1.2918070546383, 1.2918070553178, 1.07623094945772, 0.771655478205971, + 0.771655483333827, 1.07623095277605, 1.2918070612426, 1.29180706380607, + 1.07623095806282, 0.77165548712907, 0.771655491576999, 1.07623096220337, + 1.29180707050858, 1.29180707275935, 1.07623096303872, 0.771655493061662, + 0.771655488096436, 1.07623095935428, 1.29180706584451, 1.29180706091197, + 1.07623095174483, 0.771655482204467, 0.771655476539703, 1.07623094762526, + 1.29180705371894, 1.2918070538351, 1.07623094619881, 0.771655475982092, + 3.43713676411029, 3.49962488695124, 3.39577857655324, 3.22380485775483, + 3.14070016865676, 3.27729424892005, 2.88592335363144, 3.05128051782405, + 3.05128051957792, 2.88592335841908, 2.68698337060877, 2.68698336799196, + 3.49962488024322, 3.43713675823407, 3.27729424299861, 3.14070015965667, + 3.22380484741502, 3.39577856710617, 3.65861887496794, 3.69666641108682, + 3.77954577923313, 3.8123116754869, 3.7905171781339, 3.71291500353373, + 3.71291500461778, 3.65861887776378, 3.79051717887979, 3.81231167643212, + 3.77954578151993, 3.6966664143527, 3.65861887776378, 3.6966664143527, + 3.77954578151993, 3.81231167643212, 3.79051717887978, 3.71291500461778, + 3.71291500353374, 3.7905171781339, 3.81231167548691, 3.77954577923312, + 3.69666641108682, 3.65861887496794, 3.43713675823408, 3.49962488024323, + 3.39577856710617, 3.22380484741503, 3.14070015965667, 3.27729424299862, + 3.05128051782406, 2.88592335363145, 2.68698336799197, 2.68698337060877, + 2.88592335841909, 3.05128051957793, 3.27729424892005, 3.43713676411029, + 3.14070016865677, 3.22380485775483, 3.39577857655324, 3.49962488695124, + 3.79051717980798, 3.71291500595043, 3.65861887922611, 3.69666641520302, + 3.77954578211128, 3.81231167698461, 3.69666641359205, 3.65861887797203, + 3.71291500542055, 3.79051717945454, 3.81231167644173, 3.77954578099048, + 3.14070016717738, 3.2772942488296, 3.43713676345293, 3.49962488482424, + 3.39577857302032, 3.22380485410075, 2.88592336299733, 3.05128052482443, + 3.0512805236277, 2.88592336141155, 2.68698337528083, 2.686983377125, + 3.2772942517069, 3.43713676607314, 3.14070017107196, 3.22380485941718, + 3.39577857766596, 3.49962488811077, 3.60710580370843, 3.67635828474096, + 3.76574272955737, 3.79577624852149, 3.7574028163651, 3.6644417983105, + 3.79577624763033, 3.7574028157606, 3.76574272789331, 3.6763582824941, + 3.60710580188714, 3.66444179750507, 2.88150376323689, 2.78666042350304, + 2.55573223277098, 2.38227207627246, 2.48760942421827, 2.73664139789877, + 2.38227208099292, 2.48760942630927, 2.55573223893605, 2.78666042924593, + 2.88150376677042, 2.73664139983553, 3.76574272689076, 3.67635828108227, + 3.60710579922893, 3.66444179533085, 3.75740281419147, 3.79577624688206, + 3.85454389952957, 3.84162713047746, 3.85934573156789, 3.85247873751497, + 3.859345731605, 3.85454389946519, 3.85454389979809, 3.84162713106653, + 3.85934573142924, 3.85247873734865, 3.85934573150589, 3.8545438996606, + 3.85934573183683, 3.85247873823256, 3.859345731896, 3.85454389888442, + 3.84162712973063, 3.85454389900258, 3.66444179380486, 3.75740281310853, + 3.60710579835928, 3.67635827990139, 3.76574272607734, 3.79577624582888, + 3.66444179558235, 3.75740281438795, 3.60710579982856, 3.67635828188469, + 3.76574272751114, 3.79577624721842, 2.88150375660528, 2.78666041752384, + 2.55573222606169, 2.38227206890193, 2.48760941595645, 2.73664139047317, + 2.3822720705657, 2.48760941657267, 2.55573222819302, 2.78666041906537, + 2.88150375726034, 2.73664139087425, 3.79577624534713, 3.75740281296004, + 3.76574272552478, 3.67635827912559, 3.60710579807384, 3.66444179359528, + 1.94256929533975, 2.06419703422539, 1.64625341329628, 1.49258032362887, + 1.64625341618252, 1.94256929819578, 1.94256928652913, 2.06419702451545, + 1.64625340375581, 1.49258031377706, 1.64625340573984, 1.94256928791764, + 3.7657427313086, 3.67635828714392, 3.60710580747812, 3.66444180217202, + 3.75740281922419, 3.79577625024055, 3.85454390039865, 3.84162713238925, + 3.85934573111434, 3.85247873669336, 3.8593457311224, 3.85454390037747, + 3.66444180228992, 3.75740281930348, 3.60710580786064, 3.67635828756557, + 3.76574273159901, 3.79577625037159, 3.85934573200796, 3.85247873855185, + 3.85934573196111, 3.85454389875805, 3.84162712938197, 3.8545438986714, + 3.6644417924823, 3.75740281213609, 3.60710579668096, 3.67635827805925, + 3.76574272474145, 3.79577624472912, 3.79577624523555, 3.75740281245502, + 3.76574272559159, 3.67635827923429, 3.60710579757978, 3.66444179291844, + 2.88150375540616, 2.78666041686582, 2.55573222538432, 2.38227206724844, + 2.48760941375912, 2.73664138838674, 2.38227206619357, 2.48760941318099, + 2.55573222394602, 2.78666041513284, 2.88150375406378, 2.7366413877045, + 1.94256928368544, 2.06419702117996, 1.64625340097009, 1.49258030946353, + 1.64625340095025, 1.94256928336294, 1.64625340403913, 1.49258031157361, + 1.64625340140512, 1.94256928417001, 2.06419702248973, 1.9425692861929, + 2.73664138992618, 2.48760941579546, 2.88150375806085, 2.78666041976306, + 2.5557322293941, 2.3822720699552, 2.55573224014291, 2.78666043118354, + 2.88150377196666, 2.7366414061738, 2.48760943289103, 2.38227208328737, + 1.94256930302161, 2.06419704184882, 1.64625342168003, 1.49258032708336, + 1.64625341877086, 1.94256930089721, 2.73664140715473, 2.48760943439797, + 2.88150377390757, 2.78666043406952, 2.55573224445283, 2.38227208623046, + 2.3822720671636, 2.48760941446217, 2.55573222531352, 2.7866604169237, + 2.8815037561134, 2.73664138903275, 3.76574272751114, 3.79577624721842, + 3.67635828188469, 3.60710579982857, 3.66444179558235, 3.75740281438796, + 3.85454389946519, 3.84162713047747, 3.859345731605, 3.85247873751497, + 3.85934573156789, 3.85454389952957, 3.76574272607734, 3.67635827990139, + 3.60710579835928, 3.66444179380487, 3.75740281310853, 3.79577624582888, + 3.85454389888442, 3.84162712973063, 3.859345731896, 3.85247873823256, + 3.85934573183683, 3.85454389900258, 3.85934573150589, 3.85247873734865, + 3.85934573142924, 3.85454389979809, 3.84162713106653, 3.8545438996606, + 3.66444179750507, 3.7574028157606, 3.60710580188714, 3.6763582824941, + 3.76574272789332, 3.79577624763033, 3.66444179533086, 3.75740281419148, + 3.60710579922895, 3.67635828108227, 3.76574272689076, 3.79577624688206, + 2.73664139983553, 2.48760942630926, 2.8815037667704, 2.78666042924592, + 2.55573223893604, 2.38227208099292, 2.38227207627247, 2.48760942421826, + 2.55573223277098, 2.78666042350305, 2.88150376323689, 2.73664139789877, + 3.79577624852149, 3.7574028163651, 3.76574272955737, 3.67635828474096, + 3.60710580370842, 3.6644417983105, 1.64625340573983, 1.49258031377706, + 1.64625340375579, 1.94256928652912, 2.06419702451544, 1.94256928791765, + 2.73664139087425, 2.48760941657266, 2.88150375726033, 2.78666041906536, + 2.55573222819301, 2.38227207056571, 1.94256929819578, 2.06419703422538, + 1.64625341618251, 1.49258032362887, 1.64625341329627, 1.94256929533975, + 3.66444179359528, 3.75740281296004, 3.60710579807383, 3.67635827912559, + 3.76574272552478, 3.79577624534713, 2.38227206890191, 2.48760941595643, + 2.55573222606167, 2.78666041752383, 2.88150375660526, 2.73664139047317, + 3.8593457311224, 3.85247873669336, 3.85934573111434, 3.85454390039864, + 3.84162713238925, 3.85454390037747, 3.66444180217202, 3.75740281922419, + 3.60710580747812, 3.67635828714392, 3.7657427313086, 3.79577625024055, + 1.64625340095026, 1.94256928336295, 1.49258030946354, 1.64625340097011, + 1.94256928368545, 2.06419702117997, 2.7366413877045, 2.48760941318101, + 2.88150375406379, 2.78666041513284, 2.55573222394604, 2.38227206619358, + 2.38227206724845, 2.48760941375913, 2.55573222538432, 2.78666041686584, + 2.88150375540617, 2.73664138838674, 3.67635827923429, 3.76574272559159, + 3.79577624523555, 3.75740281245502, 3.66444179291844, 3.60710579757979, + 3.79577624472912, 3.75740281213609, 3.76574272474145, 3.67635827805925, + 3.60710579668096, 3.6644417924823, 3.85454389875805, 3.84162712938197, + 3.85934573196111, 3.85247873855186, 3.85934573200796, 3.8545438986714, + 3.79577625037158, 3.75740281930348, 3.76574273159901, 3.67635828756557, + 3.60710580786063, 3.66444180228992, 2.7366414061738, 2.48760943289103, + 2.88150377196667, 2.78666043118355, 2.55573224014291, 2.38227208328738, + 2.38227208623045, 2.48760943439797, 2.55573224445283, 2.78666043406951, + 2.88150377390757, 2.73664140715472, 1.9425693008972, 2.06419704184881, + 1.64625341877087, 1.49258032708335, 1.64625342168003, 1.94256930302161, + 1.64625340140513, 1.49258031157358, 1.64625340403914, 1.94256928619291, + 2.06419702248974, 1.94256928417, 2.73664138903273, 2.48760941446219, + 2.8815037561134, 2.7866604169237, 2.55573222531353, 2.38227206716359, + 2.3822720699552, 2.48760941579547, 2.5557322293941, 2.78666041976306, + 2.88150375806086, 2.73664138992617, 3.53977331774956, 3.21649654835339, + 3.21649654959376, 3.53977331919744, 3.67793587513922, 3.21649654540194, + 3.53977331564885, 3.67793587277307, 3.5397733149207, 3.21649654477022, + 0.593850783950195, 0.593850779396398, 0.593850782775934, + 0.593850791659571, 0.593850791598252, 3.67793587513923, 3.53977331919744, + 3.21649654959376, 3.21649654835339, 3.53977331774956, 3.53977331564884, + 3.21649654540194, 3.21649654477022, 3.5397733149207, 3.67793587277307, + 3.21649655924389, 3.21649655790655, 3.53977332472888, 3.67793588059418, + 3.53977332629927, 3.21649654702879, 3.53977331720081, 3.6779358746755, + 3.53977331812552, 3.21649654752238, 3.53977331812553, 3.6779358746755, + 3.53977331720082, 3.21649654702879, 3.21649654752239, 0.59385078277592, + 0.593850779396409, 0.593850783950208, 0.593850791598261, + 0.593850791659587, 3.21649655188639, 3.53977332093464, 3.6779358784194, + 3.53977332442296, 3.21649655569418, 3.6779358784194, 3.53977332093465, + 3.21649655188639, 3.21649655569419, 3.53977332442296, 3.53977332472888, + 3.21649655790655, 3.2164965592439, 3.53977332629928, 3.67793588059419, + 3.09313226062383, 2.85913196185193, 3.09313226370159, 3.47211595370855, + 3.59954851044558, 3.47211595146765, 3.80410697369584, 3.84328455748379, + 3.85117147164709, 3.80410697328136, 3.84328455709745, 3.85117147178522, + 3.85117147158168, 3.84328455729499, 3.80410697400469, 3.85117147224626, + 3.84328455656832, 3.80410697261409, 3.47211594706444, 3.59954850580681, + 3.4721159467236, 3.09313225474967, 2.85913195368211, 3.09313225521847, + 2.49541414306506, 2.11642434923235, 1.57437664082969, 1.57437664453741, + 2.11642435506745, 2.4954141451353, 3.80410697156069, 3.84328455632404, + 3.85117147267505, 3.80410697604156, 3.84328455888855, 3.85117147088586, + 3.85117147081093, 3.84328455894811, 3.8041069763742, 3.85117147291742, + 3.84328455598842, 3.80410697116346, 3.47211594499978, 3.59954850464263, + 3.4721159461514, 3.09313225360166, 2.85913195134544, 3.09313225235955, + 2.49541413793282, 2.11642434460638, 1.57437663462051, 1.57437663650702, + 2.11642434687757, 2.49541413924946, 2.11642436178007, 1.57437664731759, + 1.57437664054781, 2.11642434882777, 2.49541414518076, 2.49541415071087, + 1.57437665272758, 2.11642436593707, 2.49541415619563, 2.49541415444931, + 2.11642435990732, 1.57437665132578, 1.57437663439456, 1.57437663476153, + 2.11642434521013, 2.49541413934613, 2.4954141391958, 2.11642434512195, + 3.84328455729499, 3.85117147158168, 3.80410697261409, 3.84328455656833, + 3.85117147224626, 3.80410697400469, 3.85117147178522, 3.84328455709745, + 3.80410697328136, 3.85117147164709, 3.84328455748379, 3.80410697369584, + 2.85913196185193, 3.09313226062383, 3.47211595146765, 3.59954851044558, + 3.47211595370855, 3.09313226370158, 2.49541414513532, 2.11642435506745, + 1.57437664453741, 1.5743766408297, 2.11642434923235, 2.49541414306506, + 3.09313225521847, 2.8591319536821, 3.09313225474966, 3.47211594672359, + 3.59954850580681, 3.47211594706444, 3.84328455632404, 3.80410697156069, + 3.85117147088586, 3.84328455888855, 3.80410697604156, 3.85117147267505, + 2.1164243446064, 2.49541413793282, 2.49541413924946, 2.11642434687755, + 1.57437663650702, 1.5743766346205, 3.09313225235954, 2.85913195134545, + 3.09313225360167, 3.4721159461514, 3.59954850464264, 3.47211594499978, + 3.80410697116346, 3.84328455598842, 3.85117147291742, 3.8041069763742, + 3.84328455894811, 3.85117147081093, 2.85913197060393, 3.09313227004965, + 3.47211595876244, 3.59954851569115, 3.47211595823572, 3.09313226885001, + 2.49541415619561, 2.11642436593707, 1.57437665272757, 1.57437665132577, + 2.1164243599073, 2.4954141544493, 3.80410697522113, 3.84328455797478, + 3.85117147135552, 3.80410697204667, 3.84328455626233, 3.85117147268263, + 2.85913195335129, 3.09313225443108, 3.47211594813884, 3.59954850695455, + 3.47211594893677, 3.09313225559275, 2.11642434512197, 2.49541413919581, + 2.49541413934613, 2.11642434521014, 1.57437663439455, 1.57437663476152, + 3.09313225443106, 2.85913195335131, 3.09313225559274, 3.47211594893677, + 3.59954850695453, 3.47211594813884, 3.85117147135552, 3.80410697204667, + 3.84328455797478, 3.80410697522113, 3.85117147268263, 3.84328455626233, + 1.57437664731759, 2.11642436178009, 2.49541415071088, 2.49541414518077, + 2.11642434882777, 1.57437664054782, 3.47211595823571, 3.59954851569115, + 3.47211595876244, 3.09313227004964, 2.85913197060393, 3.09313226885001, + 2.9465344275923, 3.62843002484419, 3.81515503045961, 3.62843002252828, + 2.94653442409549, 2.94653442027827, 2.94653442228269, 3.62843002165761, + 3.81515502849687, 3.62843002048158, 1.17516569271611, 1.1751656948359, + 1.17516571106594, 1.17516570731716, 1.17516569707796, 2.94653442409548, + 3.62843002252828, 3.8151550304596, 3.62843002484421, 2.94653442759231, + 3.81515502849687, 3.62843002165761, 2.94653442228267, 2.94653442027828, + 3.62843002048158, 3.62843003086443, 2.94653443870383, 2.94653443438414, + 3.62843002819084, 3.81515503282455, 3.62843002199762, 3.81515503007801, + 3.62843002318803, 2.94653442262583, 2.94653442250223, 3.62843002199762, + 2.94653442250223, 2.94653442262583, 3.62843002318802, 3.81515503007802, + 1.17516569271612, 1.17516569707796, 1.17516570731713, 1.17516571106595, + 1.17516569483589, 3.62843002999132, 2.94653443494701, 2.94653442666621, + 3.62843002450252, 3.81515503209295, 3.62843002999132, 3.81515503209295, + 3.62843002450252, 2.94653442666622, 2.94653443494701, 3.62843003086443, + 3.81515503282455, 3.62843002819084, 2.94653443438416, 2.94653443870384, + 3.79252788302589, 2.34391114276879, 3.79252788191438, 3.79252788048945, + 2.34391113374791, 3.79252788545175, 3.79252787984217, 2.3439111305481, + 2.3439111317078, 2.34391114991211, 3.79252788191438, 3.79252788048945, + 3.79252788302589, 2.34391114276878, 2.34391113374791, 3.79252788545175, + 2.34391113054811, 3.79252787984217, 2.34391114991211, 2.34391113170778 ; + + vVelocity = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ; + + iceAreaCell = 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, + 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, + 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0 ; + + iceVolumeCell = 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, + 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, + 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0 ; + + iceAreaCategory = + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 0, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0 ; + + iceVolumeCategory = + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 0, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0 ; +} diff --git a/src/pmpo_MPMesh_assembly.hpp b/src/pmpo_MPMesh_assembly.hpp index c577506..c325a10 100644 --- a/src/pmpo_MPMesh_assembly.hpp +++ b/src/pmpo_MPMesh_assembly.hpp @@ -100,6 +100,7 @@ void MPMesh::assemblyElm0() { template void MPMesh::assemblyVtx1() { + //Mesh Information auto elm2VtxConn = p_mesh->getElm2VtxConn(); int numVtx = p_mesh->getNumVertices(); @@ -124,12 +125,15 @@ void MPMesh::assemblyVtx1() { //Reconstructed values Kokkos::View reconVals("meshField", p_mesh->getNumVertices(), numEntries); - + //Earth Radius double radius = 1.0; if(p_mesh->getGeomType() == geom_spherical_surf) radius=p_mesh->getSphereRadius(); + bool scaling=true; + int reg_method = 2; + //Assemble matrix for each vertex auto assemble = PS_LAMBDA(const int& elm, const int& mp, const int& mask) { if(mask) { //if material point is 'active'/'enabled' @@ -138,48 +142,69 @@ void MPMesh::assemblyVtx1() { int vID = elm2VtxConn(elm,i+1)-1; //vID = vertex id double w_vtx=weight(mp,i); + double mScale=1; + if(scaling) + mScale=sqrt(dual_triangle_area(vID,0))/radius; + double CoordDiffs[vec4d_nEntries] = {1, (vtxCoords(vID,0) - mpPositions(mp,0))/radius, (vtxCoords(vID,1) - mpPositions(mp,1))/radius, (vtxCoords(vID,2) - mpPositions(mp,2))/radius}; - for (int k=0; kparallel_for(assemble, "assembly"); - bool regularize=true; - //Solve Ax=b for each vertex + //Solve Ax=b for each vertex and apply regularization Kokkos::View VtxCoeffs("VtxCoeffs", p_mesh->getNumVertices()); + Kokkos::parallel_for("solving Ax=b", numVtx, KOKKOS_LAMBDA(const int vtx){ Vec4d v0 = {VtxMatrices(vtx,0,0), VtxMatrices(vtx,0,1), VtxMatrices(vtx,0,2), VtxMatrices(vtx,0,3)}; Vec4d v1 = {VtxMatrices(vtx,1,0), VtxMatrices(vtx,1,1), VtxMatrices(vtx,1,2), VtxMatrices(vtx,1,3)}; Vec4d v2 = {VtxMatrices(vtx,2,0), VtxMatrices(vtx,2,1), VtxMatrices(vtx,2,2), VtxMatrices(vtx,2,3)}; Vec4d v3 = {VtxMatrices(vtx,3,0), VtxMatrices(vtx,3,1), VtxMatrices(vtx,3,2), VtxMatrices(vtx,3,3)}; - + //Define the matrices Matrix4d A = {v0,v1,v2,v3}; Matrix4d A_regularized = {v0, v1, v2, v3}; - - //Method 1 of rgularization - //Need some kind of option to choose regularization method - /* - double A_trace = A.trace(); - A_regularized.addToDiag(A_trace*1e-8); - */ - - //Method 2 of regularization - double mScale=1.0; - if(regularize){ - mScale=sqrt(dual_triangle_area(vtx,0)); - A_regularized.scaleFirstRowAndColumn(mScale); - double regParam=0.0*EPSILON*VtxMatrices(vtx,0,0) + VtxMatrices(vtx,1,1) + VtxMatrices(vtx,2,2) + VtxMatrices(vtx,3,3); - A_regularized.addToDiag(regParam); + //Regularization + switch(reg_method){ + case 0:{ + break; + } + case 1:{ + double A_trace = A.trace(); + A_regularized.addToDiag(A_trace*1e-8); + break; + } + case 2:{ + double regParam=sqrt(EPSILON)*(VtxMatrices(vtx,0,0)+VtxMatrices(vtx,1,1)+VtxMatrices(vtx,2,2)+VtxMatrices(vtx,3,3)); + A_regularized.addToDiag(regParam); + break; + } + default:{ + printf("Invalid regularization method \n"); + break; + } } + //Solve Ax=b double coeff[vec4d_nEntries]={0.0, 0.0, 0.0, 0.0}; CholeskySolve4d_UnitRHS(A_regularized, coeff); - // Undo scaling + double mScale=1; + if(scaling) + mScale=sqrt(dual_triangle_area(vtx,0))/radius; + coeff[0]=coeff[0]*mScale*mScale; coeff[1]=coeff[1]*mScale; coeff[2]=coeff[2]*mScale; diff --git a/src/pmpo_utils.hpp b/src/pmpo_utils.hpp index 4bd4f88..7a2fdcd 100644 --- a/src/pmpo_utils.hpp +++ b/src/pmpo_utils.hpp @@ -370,27 +370,24 @@ void QRDecomp4d(Matrix4d& A, Matrix4d& Q, Matrix4d& R){ Vec4d u_column; for (int l=0; l TEST_VAL-TOLERANCE, "Error: wrong vtx mass") + !call assert(meshVtxMass(i) < TEST_VAL+TOLERANCE .and. meshVtxMass(i) > TEST_VAL-TOLERANCE, "Error: wrong vtx mass") + write(*, *) 'The value of LRV is:', meshVtxMass(i) end do !Test vtx order 1 reconstruction From bc6d2783c0671a42a600e898ca339d875286543b Mon Sep 17 00:00:00 2001 From: dhyan1272 Date: Mon, 10 Feb 2025 10:42:34 -0800 Subject: [PATCH 03/42] Tests read the areaTriangle field, so tests should be successful with regularization --- test/readMPAS.f90 | 42 ++++++++++++++++++++++++++-- test/testFortranMPReconstruction.f90 | 24 +++++++++++----- 2 files changed, 56 insertions(+), 10 deletions(-) diff --git a/test/readMPAS.f90 b/test/readMPAS.f90 index ac7549d..c5a4116 100644 --- a/test/readMPAS.f90 +++ b/test/readMPAS.f90 @@ -27,7 +27,7 @@ subroutine loadMPASMeshInPolyMPO(mpMesh, maxEdges, vertexDegree, & xVertex, yVertex, zVertex, & latVertex, & xCell, yCell, zCell, & - verticesOnCell, cellsOnCell) + verticesOnCell, cellsOnCell, areaTriangle) use :: netcdf use :: iso_c_binding implicit none @@ -42,6 +42,8 @@ subroutine loadMPASMeshInPolyMPO(mpMesh, maxEdges, vertexDegree, & real(kind=MPAS_RKIND), dimension(:), pointer :: latVertex, lonVertex real(kind=MPAS_RKIND), dimension(:), pointer :: xCell, yCell, zCell integer, dimension(:,:), pointer :: verticesOnCell, cellsOnCell + real(kind=MPAS_RKIND), dimension(:), pointer, optional :: areaTriangle + call polympo_checkPrecisionForRealKind(MPAS_RKIND) !check on maxEdges and vertexDegree @@ -75,6 +77,11 @@ subroutine loadMPASMeshInPolyMPO(mpMesh, maxEdges, vertexDegree, & !set mesh element center call polympo_setMeshElmCenter(mpMesh,nCells,c_loc(xCell),c_loc(yCell),c_loc(zCell)) + + if (present(areaTriangle)) then + call polympo_setMeshDualTriangleArea(mpMesh, nVertices, c_loc(areaTriangle)) + end if + end subroutine subroutine readMPASMeshFromNCFile(filename, maxEdges, vertexDegree, & @@ -83,7 +90,8 @@ subroutine readMPASMeshFromNCFile(filename, maxEdges, vertexDegree, & xVertex, yVertex, zVertex, & latVertex, lonVertex, & xCell, yCell, zCell, & - verticesOnCell, cellsOnCell) + verticesOnCell, cellsOnCell, & + areaTriangle) use :: netcdf use :: iso_c_binding implicit none @@ -99,12 +107,14 @@ subroutine readMPASMeshFromNCFile(filename, maxEdges, vertexDegree, & real(kind=MPAS_RKIND), dimension(:), pointer :: latVertex, lonVertex real(kind=MPAS_RKIND), dimension(:), pointer :: xCell, yCell, zCell integer, dimension(:,:), pointer :: verticesOnCell, cellsOnCell + + real(kind=MPAS_RKIND), dimension(:), pointer, optional :: areaTriangle(:) integer :: ncid, status, nCellsID, nVerticesID, maxEdgesID, vertexDegreeID, & nEdgesOnCellID, xVertexID, yVertexID, zVertexID, & latVertexID, lonVertexID, & xCellID, yCellID, zCellID, & - verticesOnCellID, cellsOnCellID + verticesOnCellID, cellsOnCellID, areaTriangleID status = nf90_open(path=trim(filename), mode=nf90_nowrite, ncid=ncid) if (status /= nf90_noerr) then @@ -181,6 +191,10 @@ subroutine readMPASMeshFromNCFile(filename, maxEdges, vertexDegree, & allocate(verticesOnCell(maxEdges, nCells)) allocate(cellsOnCell(maxEdges, nCells)) + if (present(areaTriangle)) then + allocate(areaTriangle(nVertices)) + end if + status = nf90_get_att(ncid, nf90_global, "on_a_sphere", onSphere) if (status /= nf90_noerr) then write(0, *) "readMPASMeshFromNCFile: Error on get attribute 'on_a_sphere'" @@ -275,6 +289,16 @@ subroutine readMPASMeshFromNCFile(filename, maxEdges, vertexDegree, & stop end if + if (present(areaTriangle)) then + status = nf90_inq_varid(ncid, 'areaTriangle', areaTriangleID) + if (status /= nf90_noerr) then + write(0, *) "readMPASMeshFromNCFile: Error on inquire varid of 'areaTriangleID'" + write(0, *) trim(nf90_strerror(status)) + stop + end if + end if + + !Reading the variables status = nf90_get_var(ncid, xVertexID, xVertex) if (status /= nf90_noerr) then write(0, *) "readMPASMeshFromNCFile: Error on get var of 'xVertex'" @@ -352,6 +376,18 @@ subroutine readMPASMeshFromNCFile(filename, maxEdges, vertexDegree, & stop end if + if (present(areaTriangle)) then + print *, "Reading area of dual triangles" + status = nf90_get_var(ncid, areaTriangleID, areaTriangle) + if (status /= nf90_noerr) then + write(0, *) "readMPASMeshFromNCFile: Error on get var of 'areaTriangle'" + write(0, *) trim(nf90_strerror(status)) + stop + end if + else + print *, "Not reading area of dual cells" + end if + status = nf90_close(ncid) end subroutine readMPASMeshFromNCFile subroutine setWithMPASMeshByFortran(mpMesh, fileName, n) bind(C, name="setWithMPASMeshByFortran") diff --git a/test/testFortranMPReconstruction.f90 b/test/testFortranMPReconstruction.f90 index df66815..1253265 100644 --- a/test/testFortranMPReconstruction.f90 +++ b/test/testFortranMPReconstruction.f90 @@ -19,7 +19,7 @@ program main real(kind=MPAS_RKIND) :: pi = 4.0_MPAS_RKIND*atan(1.0_MPAS_RKIND) real(kind=MPAS_RKIND) :: TEST_VAL = 1.1_MPAS_RKIND real(kind=MPAS_RKIND) :: TOLERANCE = 0.0001_MPAS_RKIND - real(kind=MPAS_RKIND) :: TOLERANCE1 = 0.0000000001_MPAS_RKIND + real(kind=MPAS_RKIND) :: TOLERANCE1 = 0.000000001_MPAS_RKIND character (len=2048) :: filename real(kind=MPAS_RKIND), dimension(:,:), pointer :: dispIncr character (len=64) :: onSphere @@ -28,12 +28,15 @@ program main real(kind=MPAS_RKIND), dimension(:), pointer :: xVertex, yVertex, zVertex real(kind=MPAS_RKIND), dimension(:), pointer :: latVertex, lonVertex real(kind=MPAS_RKIND), dimension(:), pointer :: xCell, yCell, zCell + real(kind=MPAS_RKIND), dimension(:), pointer :: areaTriangle + integer, dimension(:,:), pointer :: verticesOnCell, cellsOnCell integer :: numMPs, vID integer, dimension(:), pointer :: mpsPerElm, mp2Elm, isMPActive real(kind=MPAS_RKIND), dimension(:,:), pointer :: mpPosition, mpLatLon real(kind=MPAS_RKIND), dimension(:,:), pointer :: mpMass, mpVel real(kind=MPAS_RKIND), dimension(:), pointer :: meshVtxMass, meshElmMass, meshVtxMass1 + real(kind=MPAS_RKIND), dimension(:), pointer :: meshVtxVelu, meshVtxVelv logical :: inBound integer, parameter :: MP_ACTIVE = 1 integer, parameter :: MP_INACTIVE = 0 @@ -58,7 +61,7 @@ program main xVertex, yVertex, zVertex, & latVertex, lonVertex, & xCell, yCell, zCell, & - verticesOnCell, cellsOnCell) + verticesOnCell, cellsOnCell, areaTriangle) if (onSphere .ne. 'YES') then write (*,*) "The mesh is not spherical!" call exit(1) @@ -74,7 +77,7 @@ program main xVertex, yVertex, zVertex, & latVertex, & xCell, yCell, zCell, & - verticesOnCell, cellsOnCell) + verticesOnCell, cellsOnCell, areaTriangle) nCompsDisp = 2 allocate(dispIncr(nCompsDisp,nVertices)) @@ -90,6 +93,8 @@ program main allocate(meshVtxMass(nVertices)) allocate(meshVtxMass1(nVertices)) allocate(meshElmMass(nCells)) + allocate(meshVtxVelu(nVertices)) + allocate(meshVtxVelv(nVertices)) isMPActive = MP_ACTIVE !all active MPs and some changed below mpsPerElm = 1 !all elements have 1 MP and some changed below @@ -119,16 +124,21 @@ program main call polympo_getMeshVtxMass(mpMesh,nVertices,c_loc(meshVtxMass)) do i = 1, nVertices - !call assert(meshVtxMass(i) < TEST_VAL+TOLERANCE .and. meshVtxMass(i) > TEST_VAL-TOLERANCE, "Error: wrong vtx mass") - write(*, *) 'The value of LRV is:', meshVtxMass(i) + call assert(meshVtxMass(i) < TEST_VAL+TOLERANCE .and. meshVtxMass(i) > TEST_VAL-TOLERANCE, "Error: wrong vtx mass") + !write(*, *) 'The value of LRV is:', meshVtxMass(i) end do !Test vtx order 1 reconstruction call polympo_setReconstructionOfMass(mpMesh,1,polympo_getMeshFVtxType()) + call polympo_setReconstructionOfVel(mpMesh, 1, polympo_getMeshFVtxType()) call polympo_applyReconstruction(mpMesh) call polympo_getMeshVtxMass(mpMesh,nVertices,c_loc(meshVtxMass1)) + call polympo_getMeshVtxVel(mpMesh, nVertices, c_loc(meshVtxVelu), c_loc(meshVtxVelv)) do i = 1, nVertices - call assert(meshVtxMass1(i) < TEST_VAL+TOLERANCE1 .and. meshVtxMass1(i) > TEST_VAL-TOLERANCE1, "Error: wrong vtx mass LINEAR") + call assert(meshVtxMass1(i) < TEST_VAL+TOLERANCE1 .and. meshVtxMass1(i) > TEST_VAL-TOLERANCE1, "Error: wrong vtx mass order 1") + call assert(meshVtxVelu(i) < TEST_VAL+TOLERANCE1 .and. meshVtxVelu(i) > TEST_VAL-TOLERANCE1, "Error: wrong vtx velU order 1") + call assert(meshVtxVelv(i) < TEST_VAL+TOLERANCE1 .and. meshVtxVelv(i) > TEST_VAL-TOLERANCE1, "Error: wrong vtx velV order 1") + write(*, *) 'The value of LRV is:', meshVtxVelu(i)-TEST_VAL, meshVtxVelv(i)-TEST_VAL end do @@ -144,7 +154,7 @@ program main do i = 1, numMPs vID = verticesOnCell(1,mp2Elm(i)) - call assert(meshVtxMass(vID) < TEST_VAL+TOLERANCE .and. meshVtxMass(vID) > TEST_VAL-TOLERANCE, "Error: wrong vtx mass") + call assert(meshVtxMass(vID) < TEST_VAL+TOLERANCE1 .and. meshVtxMass(vID) > TEST_VAL-TOLERANCE1, "Error: wrong vtx mass push") call assert(meshElmMass(mp2Elm(i)) < TEST_VAL+TOLERANCE & .and. meshElmMass(mp2Elm(i)) > TEST_VAL-TOLERANCE, "Error: wrong elm mass") From f8d54fc355c1af658f06847a54c21a303ae95ceb Mon Sep 17 00:00:00 2001 From: dhyan1272 Date: Fri, 14 Feb 2025 09:35:11 -0800 Subject: [PATCH 04/42] Dellaocation, removed log file and created paper folder --- log_MP | 6863 -------------------------- test/testFortranMPReconstruction.f90 | 2 + 2 files changed, 2 insertions(+), 6863 deletions(-) delete mode 100644 log_MP diff --git a/log_MP b/log_MP deleted file mode 100644 index 39918fb..0000000 --- a/log_MP +++ /dev/null @@ -1,6863 +0,0 @@ -netcdf ic_slotted_cylinder_2562 { -dimensions: - nCells = 2562 ; - nVertices = 5120 ; - nCategories = 1 ; - ONE = 1 ; -variables: - double uVelocity(nVertices) ; - double vVelocity(nVertices) ; - double iceAreaCell(nCells) ; - double iceVolumeCell(nCells) ; - double iceAreaCategory(nCells, nCategories, ONE) ; - double iceVolumeCategory(nCells, nCategories, ONE) ; -data: - - uVelocity = 3.51971928314205, 3.47512234552054, 3.39605364751305, - 3.39605364776838, 3.47512234595256, 3.39605364415435, 3.47512234253748, - 3.51971928024815, 3.47512234234787, 3.39605364424297, 0.154357464161867, - 0.154357464418338, 0.154357464376962, 0.154357464500569, - 0.154357464373758, 3.39605364751305, 3.47512234552054, 3.51971928314207, - 3.47512234595257, 3.39605364776839, 3.39605364415435, 3.39605364424296, - 3.47512234234787, 3.51971928024815, 3.47512234253748, 3.39605365600016, - 3.47512235331092, 3.51971929051509, 3.47512235377981, 3.39605365628955, - 3.39605364650079, 3.4751223448265, 3.51971928232674, 3.47512234510602, - 3.3960536468386, 3.3960536465008, 3.39605364683861, 3.47512234510602, - 3.51971928232674, 3.4751223448265, 0.154357464161846, 0.154357464373768, - 0.154357464500558, 0.154357464376972, 0.154357464418328, - 3.47512234965195, 3.51971928719354, 3.47512235073297, 3.39605365277294, - 3.3960536517862, 3.39605365277296, 3.47512235073298, 3.51971928719355, - 3.47512234965196, 3.39605365178622, 3.39605365600017, 3.39605365628957, - 3.47512235377982, 3.51971929051509, 3.47512235331093, 3.33632040348494, - 3.3707523490026, 3.33632040282711, 3.23195483390815, 3.19170730176026, - 3.23195483460147, 3.86052719315628, 3.85735494265993, 3.8600357605879, - 3.86052719312186, 3.85735494257026, 3.86003576053382, 3.85735494241866, - 3.86052719307657, 3.86003576063311, 3.85735494279346, 3.86052719322088, - 3.86003576043921, 3.33632039673443, 3.37075234295757, 3.33632039667594, - 3.23195482756115, 3.19170729467244, 3.23195482762904, 2.14741220053217, - 2.14741220011114, 2.03504821319991, 1.91331103281558, 1.9133110333481, - 2.03504821501541, 3.85735494318127, 3.86003576090522, 3.86052719293465, - 3.85735494202528, 3.86003576029375, 3.86052719337439, 3.86052719290206, - 3.86003576093899, 3.85735494327473, 3.86052719341129, 3.86003576023198, - 3.85735494194176, 3.3363203952616, 3.23195482587916, 3.1917072928257, - 3.23195482556806, 3.33632039492412, 3.37075234133105, 2.14741219436931, - 2.14741219388141, 2.03504820739424, 1.91331102618635, 1.91331102674549, - 2.03504820792936, 1.91331103481011, 2.03504821492811, 2.14741220272273, - 2.14741220392819, 2.03504821887084, 1.9133110359939, 2.14741221165298, - 2.03504822395832, 1.91331104400562, 1.91331104358003, 2.03504822645895, - 2.14741221130944, 2.14741219417938, 2.03504820734298, 1.9133110261006, - 1.91331102602604, 2.03504820752655, 2.14741219418535, 3.86052719322088, - 3.85735494279346, 3.86003576063311, 3.86052719307657, 3.85735494241866, - 3.86003576043921, 3.86052719312186, 3.8600357605879, 3.85735494265993, - 3.86052719315628, 3.86003576053382, 3.85735494257026, 3.3363204028271, - 3.37075234900259, 3.33632040348493, 3.23195483460147, 3.19170730176026, - 3.23195483390815, 1.9133110333481, 1.91331103281558, 2.03504821319992, - 2.14741220011115, 2.14741220053218, 2.03504821501541, 3.33632039673443, - 3.23195482762903, 3.19170729467243, 3.23195482756114, 3.33632039667594, - 3.37075234295757, 3.86052719293466, 3.86003576090522, 3.85735494318127, - 3.86052719337439, 3.86003576029375, 3.85735494202528, 1.91331102618635, - 2.03504820739425, 2.14741219388141, 2.14741219436931, 2.03504820792935, - 1.9133110267455, 3.33632039492412, 3.23195482556806, 3.19170729282571, - 3.23195482587917, 3.3363203952616, 3.37075234133105, 3.86003576093899, - 3.86052719290206, 3.85735494194176, 3.86003576023198, 3.86052719341129, - 3.85735494327473, 3.23195484163699, 3.19170730967967, 3.23195484186681, - 3.33632041042294, 3.37075235592902, 3.33632041023479, 2.14741221165297, - 2.14741221130942, 2.03504822645893, 1.91331104358002, 1.91331104400561, - 2.03504822395829, 3.86003576034742, 3.86052719332038, 3.85735494301239, - 3.86003576077637, 3.86052719300741, 3.85735494219614, 3.33632039797719, - 3.23195482781526, 3.19170729537129, 3.23195482758278, 3.33632039772401, - 3.3707523434905, 1.91331102602603, 1.91331102610058, 2.03504820734297, - 2.14741219417937, 2.14741219418535, 2.03504820752655, 3.33632039797719, - 3.37075234349048, 3.33632039772401, 3.23195482758276, 3.19170729537129, - 3.23195482781525, 3.85735494301239, 3.86052719332038, 3.86003576034742, - 3.85735494219614, 3.86052719300741, 3.86003576077637, 2.14741220272273, - 2.03504821492812, 1.91331103481012, 1.91331103599391, 2.03504821887085, - 2.1474122039282, 3.19170730967966, 3.23195484163698, 3.33632041023479, - 3.37075235592902, 3.33632041042294, 3.23195484186681, 3.29636636362926, - 3.24773691482299, 3.27150647779161, 3.37084721086564, 3.41435559172517, - 3.39249536885371, 3.76894522317819, 3.7353246005687, 3.70681333198533, - 3.72534591930479, 3.76403894959448, 3.78623820284751, 3.7353245999012, - 3.7689452226303, 3.78623820219564, 3.76403894866395, 3.72534591818006, - 3.70681333098936, 3.27150647403733, 3.24773691135723, 3.29636636079297, - 3.39249536603282, 3.41435558865646, 3.37084720718277, 2.95756490679779, - 2.87690638401446, 2.77685515752924, 2.77685515780303, 2.87690638510949, - 2.95756490706125, 2.87690638022213, 2.95756490249572, 2.95756490235291, - 2.87690637954507, 2.7768551526422, 2.77685515289343, 3.29636635933764, - 3.24773691023361, 3.27150647312524, 3.37084720638359, 3.41435558767133, - 3.39249536475112, 3.70681332896657, 3.72534591679101, 3.76403894695546, - 3.78623820082457, 3.76894522059566, 3.73532459820695, 3.73532459794603, - 3.768945220452, 3.7862382005649, 3.76403894665404, 3.7253459162681, - 3.70681332863849, 3.29636635776033, 3.39249536309733, 3.41435558602935, - 3.37084720445461, 3.27150647116826, 3.24773690820345, 0.903438045650939, - 1.03257039861662, 1.16261876910387, 1.16261876965589, 1.03257039872572, - 0.903438046431373, 0.903438053592724, 1.03257040536273, 1.16261877767811, - 1.16261877906432, 1.03257041067264, 0.903438055183172, 0.903438061183353, - 1.03257041680396, 1.16261878592089, 1.16261878643253, 1.03257041488004, - 0.903438061661399, 1.03257040616398, 0.903438053635351, - 0.903438055078504, 1.03257040919781, 1.16261877847409, 1.16261877734117, - 1.03257039949571, 0.903438046662163, 0.903438047790002, 1.0325704004215, - 1.16261877103789, 1.1626187701149, 3.29636636079297, 3.24773691135723, - 3.27150647403733, 3.37084720718277, 3.41435558865647, 3.39249536603282, - 3.76894522263029, 3.7353245999012, 3.70681333098936, 3.72534591818006, - 3.76403894866395, 3.78623820219564, 3.7353246005687, 3.76894522317818, - 3.78623820284752, 3.76403894959447, 3.7253459193048, 3.70681333198533, - 3.247736914823, 3.29636636362927, 3.39249536885371, 3.41435559172518, - 3.37084721086565, 3.27150647779162, 2.95756490679779, 2.95756490706126, - 2.87690638510949, 2.77685515780302, 2.77685515752923, 2.87690638401446, - 3.72534591679101, 3.70681332896656, 3.73532459820695, 3.76894522059565, - 3.78623820082457, 3.76403894695546, 3.2477369102336, 3.29636635933764, - 3.39249536475111, 3.41435558767132, 3.37084720638358, 3.27150647312524, - 2.95756490249571, 2.87690638022212, 2.77685515289343, 2.77685515264219, - 2.87690637954508, 2.95756490235289, 3.29636635776033, 3.24773690820345, - 3.27150647116826, 3.37084720445462, 3.41435558602936, 3.39249536309733, - 3.73532459794603, 3.70681332863849, 3.7253459162681, 3.76403894665404, - 3.78623820056489, 3.768945220452, 3.24773692441733, 3.29636637269177, - 3.39249537736691, 3.41435559992311, 3.37084721945235, 3.27150648687924, - 2.8769063948919, 2.95756491772741, 2.95756491780715, 2.8769063964159, - 2.77685516890115, 2.77685516892971, 3.29636636938179, 3.24773692030793, - 3.27150648241272, 3.37084721516382, 3.4143555962887, 3.39249537413924, - 3.70681333568204, 3.7253459226604, 3.76403895224453, 3.78623820548746, - 3.76894522614054, 3.73532460428986, 3.70681333674755, 3.73532460502842, - 3.76894522673302, 3.78623820620996, 3.76403895324676, 3.7253459239122, - 3.70681333043198, 3.72534591774692, 3.7640389480836, 3.78623820181339, - 3.76894522210539, 3.73532459949979, 3.73532460009807, 3.76894522282055, - 3.78623820247427, 3.76403894917696, 3.72534591863172, 3.70681333147836, - 3.24773691120959, 3.29636636054265, 3.39249536635517, 3.41435558881604, - 3.37084720765732, 3.27150647379344, 2.77685515429861, 2.8769063816351, - 2.95756490432179, 2.95756490447969, 2.87690638155258, 2.77685515435747, - 3.29636635980591, 3.24773691052675, 3.27150647296846, 3.37084720645808, - 3.4143555878722, 3.39249536540832, 3.27150647296846, 3.24773691052675, - 3.29636635980591, 3.39249536540832, 3.41435558787219, 3.37084720645808, - 2.9575649043218, 2.8769063816351, 2.77685515429862, 2.77685515435748, - 2.87690638155259, 2.95756490447969, 3.29636636054265, 3.24773691120959, - 3.27150647379342, 3.37084720765732, 3.41435558881603, 3.39249536635517, - 3.76894522282056, 3.73532460009807, 3.70681333147836, 3.72534591863172, - 3.76403894917696, 3.78623820247427, 3.70681333043198, 3.73532459949979, - 3.76894522210539, 3.78623820181339, 3.76403894808361, 3.72534591774692, - 0.903438046662163, 1.03257039949573, 1.16261877011491, 1.16261877103791, - 1.03257040042149, 0.903438047790018, 0.903438053635359, 1.03257040616398, - 1.16261877734117, 1.16261877847409, 1.03257040919781, 0.903438055078513, - 1.03257041680396, 0.903438061183345, 0.903438061661404, 1.03257041488003, - 1.16261878643252, 1.16261878592087, 1.03257040536272, 0.903438053592724, - 0.903438055183179, 1.03257041067264, 1.16261877906432, 1.1626187776781, - 0.903438045650941, 0.90343804643137, 1.03257039872571, 1.1626187696559, - 1.16261876910389, 1.03257039861661, 3.24773692218824, 3.29636637051913, - 3.39249537553824, 3.41435559839331, 3.37084721805891, 3.27150648520813, - 2.77685516163244, 2.87690638833652, 2.95756491116434, 2.9575649125277, - 2.87690639049826, 2.77685516298305, 3.37084720951275, 3.4143555906345, - 3.39249536880745, 3.29636636297443, 3.24773691395421, 3.27150647558195, - 3.76894522472282, 3.73532460220802, 3.7068133334731, 3.7253459202698, - 3.76403895060766, 3.78623820394614, 3.7253459229499, 3.70681333565846, - 3.73532460382477, 3.76894522585012, 3.78623820545346, 3.76403895252925, - 3.70681333565847, 3.7253459229499, 3.76403895252925, 3.78623820545346, - 3.76894522585012, 3.73532460382476, 3.73532460220802, 3.76894522472282, - 3.78623820394614, 3.76403895060766, 3.7253459202698, 3.7068133334731, - 3.39249536880746, 3.4143555906345, 3.37084720951276, 3.27150647558194, - 3.24773691395422, 3.29636636297443, 2.95756491116436, 2.87690638833653, - 2.77685516163244, 2.77685516298306, 2.87690639049828, 2.95756491252772, - 3.29636637051912, 3.24773692218824, 3.27150648520813, 3.37084721805891, - 3.41435559839331, 3.39249537553824, 3.73532460502842, 3.70681333674755, - 3.7253459239122, 3.76403895324676, 3.78623820620996, 3.76894522673302, - 3.76403895224453, 3.7253459226604, 3.70681333568204, 3.73532460428986, - 3.76894522614055, 3.78623820548746, 3.27150648241272, 3.24773692030795, - 3.2963663693818, 3.39249537413925, 3.4143555962887, 3.37084721516383, - 2.95756491772742, 2.87690639489191, 2.77685516892972, 2.77685516890116, - 2.87690639641591, 2.95756491780717, 3.29636637269178, 3.24773692441733, - 3.27150648687924, 3.37084721945236, 3.41435559992312, 3.39249537736691, - 3.69034136284513, 3.66902586196525, 3.69610022881746, 3.74352818035439, - 3.76062802068365, 3.73904385009254, 3.73904384915359, 3.76062801951332, - 3.74352817885714, 3.69610022697635, 3.66902586039812, 3.69034136170115, - 2.68983921205122, 2.75552219944641, 2.71743080819205, 2.59000888207046, - 2.51344855277885, 2.55900622258916, 2.55900622580187, 2.5134485569339, - 2.59000888740643, 2.71743081296537, 2.75552220330374, 2.68983921494602, - 3.76062801822243, 3.7435281777863, 3.69610022525304, 3.66902585859627, - 3.69034135959205, 3.73904384779614, 3.85429643637894, 3.85870568247999, - 3.86103574461437, 3.85946486675864, 3.86103574461713, 3.85870568246069, - 3.85429643661807, 3.85870568266521, 3.86103574459901, 3.85946486664261, - 3.86103574460411, 3.85870568261925, 3.85870568218824, 3.86103574465829, - 3.85946486702695, 3.86103574466378, 3.85870568216127, 3.8542964358189, - 3.73904384643936, 3.69034135855334, 3.66902585724456, 3.6961002243344, - 3.74352817661265, 3.76062801727404, 3.73904384810876, 3.69034135995964, - 3.66902585911129, 3.69610022588481, 3.74352817832292, 3.76062801862327, - 2.68983920460133, 2.75552219278086, 2.71743080150696, 2.59000887541825, - 2.51344854530942, 2.55900621516455, 2.55900621604943, 2.51344854661861, - 2.59000887703654, 2.71743080274771, 2.75552219367367, 2.68983920527075, - 3.73904384608154, 3.7606280169476, 3.74352817596999, 3.69610022391329, - 3.66902585678567, 3.69034135833747, 1.92822120497328, 1.85158065035756, - 1.71673000672189, 1.62632047729976, 1.71673000829342, 1.8515806516817, - 1.92822119535366, 1.85158064069737, 1.71672999729685, 1.6263204671434, - 1.71672999818415, 1.85158064146224, 3.7606280230259, 3.74352818249984, - 3.69610023151787, 3.66902586514668, 3.6903413662693, 3.73904385281335, - 3.85429643730031, 3.85870568308304, 3.86103574455386, 3.85946486629979, - 3.86103574455475, 3.85870568307874, 3.73904385294298, 3.69034136645886, - 3.66902586540083, 3.69610023185968, 3.74352818274118, 3.76062802319882, - 3.85870568203261, 3.86103574468196, 3.85946486714144, 3.8610357446785, - 3.85870568205915, 3.85429643558444, 3.7390438452, 3.69034135723507, - 3.66902585564614, 3.69610022281084, 3.74352817513519, 3.76062801610617, - 3.73904384571644, 3.76062801671597, 3.74352817595134, 3.69610022373314, - 3.66902585648341, 3.69034135782162, 2.68983920291653, 2.75552219136651, - 2.71743080052465, 2.59000887428552, 2.51344854383711, 2.55900621324726, - 2.55900621230208, 2.51344854275856, 2.59000887285915, 2.7174307990401, - 2.7555221900842, 2.68983920195731, 1.92822119160674, 1.85158063755821, - 1.71672999376142, 1.62632046329999, 1.71672999368156, 1.85158063745382, - 1.85158063947254, 1.71672999611985, 1.62632046481581, 1.71672999490435, - 1.85158063836851, 1.9282211930361, 2.55900621513632, 2.68983920528556, - 2.75552219342873, 2.71743080364367, 2.5900088770865, 2.51344854670601, - 2.55900623044055, 2.51344856098806, 2.59000888952688, 2.71743081648028, - 2.75552220758498, 2.68983922087117, 1.92822121048097, 1.85158065838457, - 1.71673001255179, 1.62632048317417, 1.71673001130163, 1.851580657115, - 2.55900623226872, 2.68983922270568, 2.75552220953349, 2.71743081948777, - 2.59000889252288, 2.51344856394869, 2.55900621345203, 2.51344854401996, - 2.59000887420429, 2.71743080075885, 2.75552219160256, 2.68983920357477, - 3.73904384810876, 3.76062801862328, 3.74352817832292, 3.69610022588482, - 3.66902585911129, 3.69034135995965, 3.85429643637894, 3.8587056824607, - 3.86103574461713, 3.85946486675864, 3.86103574461437, 3.85870568247999, - 3.76062801727404, 3.74352817661265, 3.6961002243344, 3.66902585724455, - 3.69034135855335, 3.73904384643936, 3.8542964358189, 3.85870568216127, - 3.86103574466378, 3.85946486702695, 3.86103574465829, 3.85870568218824, - 3.85870568261926, 3.86103574460411, 3.85946486664261, 3.86103574459901, - 3.85870568266521, 3.85429643661808, 3.73904384915359, 3.69034136170115, - 3.66902586039812, 3.69610022697636, 3.74352817885714, 3.76062801951333, - 3.73904384779614, 3.69034135959206, 3.66902585859628, 3.69610022525305, - 3.7435281777863, 3.76062801822244, 2.55900622580186, 2.68983921494601, - 2.75552220330374, 2.71743081296536, 2.59000888740643, 2.51344855693388, - 2.55900622258917, 2.51344855277885, 2.59000888207047, 2.71743080819205, - 2.75552219944642, 2.68983921205122, 3.73904385009254, 3.76062802068365, - 3.74352818035439, 3.69610022881747, 3.66902586196525, 3.69034136284513, - 1.85158064146224, 1.71672999818415, 1.62632046714338, 1.71672999729685, - 1.85158064069735, 1.92822119535365, 2.55900621604943, 2.68983920527074, - 2.75552219367367, 2.7174308027477, 2.59000887703654, 2.51344854661859, - 1.92822120497329, 1.85158065168169, 1.71673000829342, 1.62632047729975, - 1.71673000672189, 1.85158065035755, 3.73904384608153, 3.69034135833747, - 3.66902585678566, 3.69610022391329, 3.74352817596998, 3.76062801694759, - 2.55900621516454, 2.5134485453094, 2.59000887541824, 2.71743080150694, - 2.75552219278085, 2.68983920460131, 3.85870568307874, 3.86103574455475, - 3.85946486629979, 3.86103574455386, 3.85870568308304, 3.85429643730031, - 3.73904385281335, 3.6903413662693, 3.66902586514668, 3.69610023151787, - 3.74352818249984, 3.7606280230259, 1.71672999368156, 1.62632046330001, - 1.71672999376143, 1.85158063755823, 1.92822119160676, 1.85158063745385, - 2.5590062123021, 2.68983920195732, 2.75552219008421, 2.71743079904012, - 2.59000887285916, 2.51344854275857, 2.55900621324727, 2.51344854383711, - 2.59000887428552, 2.71743080052466, 2.75552219136651, 2.68983920291653, - 3.66902585648342, 3.69610022373314, 3.74352817595135, 3.76062801671596, - 3.73904384571643, 3.69034135782162, 3.7390438452, 3.76062801610617, - 3.74352817513519, 3.69610022281084, 3.66902585564614, 3.69034135723507, - 3.85429643558444, 3.85870568205915, 3.8610357446785, 3.85946486714144, - 3.86103574468196, 3.85870568203261, 3.73904385294298, 3.76062802319882, - 3.74352818274118, 3.69610023185968, 3.66902586540083, 3.69034136645886, - 2.75552220758498, 2.71743081648029, 2.59000888952688, 2.51344856098807, - 2.55900623044056, 2.68983922087118, 2.55900623226872, 2.5134485639487, - 2.59000889252288, 2.71743081948776, 2.75552220953348, 2.68983922270568, - 1.92822121048097, 1.851580657115, 1.71673001130164, 1.62632048317417, - 1.71673001255179, 1.85158065838458, 1.85158063836852, 1.71672999490434, - 1.62632046481582, 1.71672999611984, 1.85158063947254, 1.92822119303609, - 2.55900621345202, 2.68983920357478, 2.75552219160255, 2.71743080075886, - 2.59000887420428, 2.51344854401997, 2.55900621513632, 2.51344854670602, - 2.59000887708649, 2.71743080364368, 2.75552219342872, 2.68983920528556, - 3.31181370257387, 3.35908132346166, 3.44625835690342, 3.4633277441073, - 3.41800761050411, 3.32584574099211, 3.26804799730986, 3.20942955850101, - 3.12164836499262, 3.12164836525515, 3.20942955923946, 3.26804799757476, - 3.35908132450756, 3.31181370406625, 3.32584574291193, 3.41800761247519, - 3.46332774556996, 3.44625835807049, 3.6450147615448, 3.60067298658623, - 3.56009474911326, 3.58034875564143, 3.63825080787157, 3.67297415202974, - 3.56009474835612, 3.60067298625167, 3.64501476122664, 3.67297415146641, - 3.63825080692829, 3.58034875465491, 3.3590813206922, 3.31181370020673, - 3.32584573889264, 3.41800760864817, 3.46332774205432, 3.44625835445029, - 3.56009474573959, 3.58034875259364, 3.63825080461649, 3.67297414943854, - 3.64501475846348, 3.60067298383726, 3.5803487521188, 3.56009474546507, - 3.60067298372279, 3.64501475837662, 3.6729741492274, 3.6382508042955, - 3.31181369907863, 3.35908132025738, 3.44625835374865, 3.46332774127064, - 3.41800760742445, 3.32584573785137, 3.26804799362015, 3.26804799367781, - 3.20942955460651, 3.12164836096841, 3.12164836100006, 3.20942955528946, - 0.513170980391654, 0.390101460425834, 0.39010146214344, - 0.513170982837199, 0.644390362785376, 0.644390361391548, - 0.39010145995051, 0.51317097986375, 0.644390360553818, 0.644390361614694, - 0.513170981309957, 0.390101461351891, 0.390101464472655, - 0.513170985220625, 0.644390367300331, 0.644390369210505, - 0.513170992235016, 0.390101466818805, 0.390101469340862, - 0.513170995446163, 0.644390373865335, 0.644390374285878, - 0.513170994512027, 0.390101469695325, 0.390101464823125, - 0.390101467148519, 0.513170991705255, 0.644390369430307, - 0.644390367590229, 0.513170986531185, 3.60067298658624, 3.64501476154479, - 3.67297415202976, 3.63825080787157, 3.58034875564144, 3.56009474911326, - 3.31181370406625, 3.35908132450757, 3.4462583580705, 3.46332774556998, - 3.4180076124752, 3.32584574291194, 3.20942955850101, 3.26804799730986, - 3.26804799757476, 3.20942955923946, 3.12164836525515, 3.12164836499261, - 3.35908132346166, 3.31181370257388, 3.32584574099211, 3.41800761050411, - 3.46332774410731, 3.44625835690342, 3.56009474835613, 3.58034875465492, - 3.63825080692828, 3.67297415146642, 3.64501476122664, 3.60067298625168, - 3.31181370020672, 3.35908132069219, 3.44625835445029, 3.46332774205431, - 3.41800760864816, 3.32584573889263, 3.26804799362015, 3.20942955528946, - 3.12164836100006, 3.12164836096841, 3.20942955460651, 3.26804799367781, - 3.35908132025737, 3.31181369907862, 3.32584573785136, 3.41800760742444, - 3.46332774127064, 3.44625835374865, 3.56009474546506, 3.5803487521188, - 3.6382508042955, 3.67297414922739, 3.64501475837662, 3.60067298372279, - 3.56009474573959, 3.60067298383726, 3.64501475846347, 3.67297414943854, - 3.63825080461649, 3.58034875259363, 3.20942956821056, 3.26804800673831, - 3.26804800699638, 3.20942956910064, 3.12164837543289, 3.12164837525338, - 3.35908133210083, 3.31181371157333, 3.3258457496641, 3.41800761859748, - 3.46332775179875, 3.4462583648768, 3.56009475520654, 3.58034876125019, - 3.6382508127279, 3.67297415692659, 3.64501476699879, 3.60067299271539, - 3.58034876235721, 3.56009475602123, 3.60067299309049, 3.64501476733615, - 3.67297415755316, 3.63825081373643, 3.35908133326547, 3.44625836615578, - 3.46332775346192, 3.41800762080316, 3.32584575184952, 3.31181371327306, - 3.35908132245567, 3.31181370178286, 3.32584574001317, 3.41800760989976, - 3.4633277433773, 3.44625835626692, 3.56009474779591, 3.58034875408694, - 3.63825080639685, 3.67297415100882, 3.6450147607534, 3.60067298565548, - 3.600672985903, 3.64501476108012, 3.67297415145966, 3.6382508072732, - 3.58034875468341, 3.5600947484059, 3.31181370203029, 3.35908132303664, - 3.44625835680565, 3.46332774403365, 3.41800761064153, 3.32584574069303, - 3.26804799594899, 3.26804799628088, 3.20942955723016, 3.12164836340318, - 3.12164836314393, 3.20942955730212, 3.64501476108012, 3.60067298590299, - 3.5600947484059, 3.58034875468341, 3.63825080727321, 3.67297415145966, - 3.58034875408695, 3.56009474779592, 3.60067298565547, 3.64501476075342, - 3.67297415100882, 3.63825080639685, 3.31181370178285, 3.35908132245566, - 3.44625835626693, 3.4633277433773, 3.41800760989977, 3.32584574001317, - 3.26804799594899, 3.20942955730213, 3.12164836314393, 3.12164836340318, - 3.20942955723017, 3.26804799628088, 3.35908132303663, 3.31181370203028, - 3.32584574069302, 3.41800761064153, 3.46332774403365, 3.44625835680565, - 0.513170979863757, 0.390101459950514, 0.390101461351887, - 0.513170981309937, 0.644390361614679, 0.644390360553805, - 0.390101460425834, 0.513170980391677, 0.644390361391548, - 0.644390362785384, 0.513170982837199, 0.390101462143453, - 0.390101464823112, 0.513170986531194, 0.644390367590255, - 0.64439036943033, 0.513170991705272, 0.390101467148507, - 0.513170995446179, 0.390101469340858, 0.390101469695334, - 0.513170994512023, 0.644390374285886, 0.644390373865327, - 0.39010146447265, 0.390101466818792, 0.513170992235013, - 0.644390369210495, 0.644390367300323, 0.513170985220632, - 3.46332774715531, 3.44625836059314, 3.3590813270382, 3.3118137064222, - 3.32584574385911, 3.41800761373586, 3.64501476467293, 3.60067298969088, - 3.56009475193647, 3.58034875769823, 3.63825080995988, 3.67297415433584, - 3.58034876026232, 3.56009475377362, 3.60067299058148, 3.64501476540208, - 3.67297415576365, 3.63825081214336, 3.31181370995532, 3.35908133015054, - 3.44625836348736, 3.46332775106351, 3.41800761854248, 3.32584574918805, - 3.26804800175343, 3.26804800296036, 3.20942956456826, 3.12164837079488, - 3.12164836947269, 3.2094295631454, 3.60067298969089, 3.64501476467293, - 3.67297415433583, 3.63825080995988, 3.58034875769823, 3.56009475193648, - 3.44625836059316, 3.46332774715532, 3.41800761373587, 3.32584574385913, - 3.31181370642221, 3.35908132703821, 3.26804800175343, 3.20942956314541, - 3.1216483694727, 3.12164837079489, 3.20942956456827, 3.26804800296038, - 3.35908133015054, 3.31181370995533, 3.32584574918805, 3.41800761854248, - 3.46332775106352, 3.44625836348737, 3.56009475377363, 3.58034876026232, - 3.63825081214337, 3.67297415576365, 3.64501476540209, 3.60067299058148, - 3.31181371157334, 3.35908133210083, 3.44625836487681, 3.46332775179875, - 3.41800761859748, 3.3258457496641, 3.26804800673831, 3.20942956821056, - 3.12164837525338, 3.1216483754329, 3.20942956910064, 3.26804800699638, - 3.35908133326547, 3.31181371327306, 3.32584575184952, 3.41800762080316, - 3.46332775346192, 3.44625836615579, 3.56009475602124, 3.58034876235721, - 3.63825081373643, 3.67297415755317, 3.64501476733615, 3.6006729930905, - 3.56009475520653, 3.60067299271539, 3.6450147669988, 3.6729741569266, - 3.63825081272791, 3.58034876125018, 3.08082660801717, 3.04241173682454, - 2.9206912327572, 2.86476252917547, 2.9105012306686, 3.0334125478374, - 2.9105012315398, 2.86476253095996, 2.92069123535957, 3.04241173925898, - 3.08082660957038, 3.03341254865553, 3.35389278326266, 3.38203646367467, - 3.34224676909528, 3.2387419855312, 3.20616415863302, 3.25207929631784, - 3.46488607518047, 3.49924465455745, 3.57603285089435, 3.60145794013974, - 3.57210180618271, 3.49454781110009, 3.57210180569235, 3.60145793920912, - 3.57603284944596, 3.49924465294645, 3.46488607403731, 3.49454781055657, - 3.38203646125342, 3.35389278008579, 3.25207929293554, 3.20616415572886, - 3.2387419834683, 3.34224676714462, 3.83730049868562, 3.81602439422394, - 3.79952374854542, 3.81216850450131, 3.83475040255377, 3.84515172371925, - 3.85524174976196, 3.85898334461991, 3.86098785453913, 3.85897089845091, - 3.8608413352106, 3.85946975252995, 3.84228359978211, 3.82457407797039, - 3.80871340093741, 3.82041520176199, 3.8399968010498, 3.8495135572642, - 3.83730049842567, 3.81602439371894, 3.79952374798096, 3.81216850381735, - 3.83475040220811, 3.84515172343446, 3.85524174961272, 3.85898334449836, - 3.86098785455535, 3.85897089853854, 3.8608413352336, 3.85946975244585, - 3.84228359998145, 3.82457407823984, 3.80871340125699, 3.8204152021239, - 3.83999680125938, 3.84951355742011, 3.86084133519797, 3.85897089845485, - 3.86098785454272, 3.85898334455374, 3.8552417497379, 3.8594697525296, - 3.83475040251148, 3.81216850420118, 3.79952374850035, 3.81602439426909, - 3.8373004988564, 3.84515172372913, 3.82457407890996, 3.84228360050848, - 3.8495135577529, 3.83999680161838, 3.82041520249319, 3.808713401902, - 3.86084133529588, 3.85897089874415, 3.86098785460524, 3.85898334428236, - 3.85524174925527, 3.85946975229149, 3.83475040154957, 3.81216850309196, - 3.79952374711499, 3.81602439313399, 3.83730049793289, 3.84515172301957, - 3.82457407770568, 3.84228359963091, 3.84951355708345, 3.83999680072582, - 3.82041520132951, 3.80871340048797, 3.4648860689349, 3.49924464850608, - 3.57603284532914, 3.60145793511001, 3.5721018009682, 3.49454780548832, - 3.57210180088351, 3.60145793499067, 3.57603284505017, 3.49924464832333, - 3.46488606879208, 3.49454780543662, 3.25207928778574, 3.20616415000751, - 3.23874197774547, 3.34224676160499, 3.38203645632499, 3.35389277511682, - 3.080826601482, 3.04241173027962, 2.92069122612521, 2.86476252177829, - 2.91050122330073, 3.03341254054834, 2.91050122343423, 2.86476252206779, - 2.92069122656977, 3.04241173059118, 3.08082660167768, 3.03341254063997, - 3.35389277555892, 3.38203645662013, 3.34224676181183, 3.23874197798445, - 3.20616415034957, 3.25207928829333, 2.28022860418703, 2.40265427515554, - 2.4295968822255, 2.34121441610718, 2.21188697166056, 2.17953696695939, - 1.9849406019266, 1.88375469162434, 1.75116085122506, 1.70415170279435, - 1.81775135254054, 1.94602812550664, 1.41889854835934, 1.54672376368563, - 1.66983705266571, 1.66983705198026, 1.54672376143888, 1.41889854747539, - 1.81775135569696, 1.704151707568, 1.75116085710719, 1.88375469823445, - 1.98494060702109, 1.94602812971535, 2.21188697610915, 2.34121442127904, - 2.42959688613418, 2.40265427847282, 2.28022860654878, 2.17953697069596, - 2.58069898330844, 2.48250524607293, 2.37040346111973, 2.37040346146522, - 2.48250524752885, 2.58069898360735, 3.83730049722269, 3.81602439252289, - 3.79952374635626, 3.81216850268673, 3.83475040104545, 3.84515172264257, - 3.85524174902513, 3.85898334420375, 3.86098785463037, 3.85897089887936, - 3.86084133536623, 3.85946975217001, 3.80871340338234, 3.82041520419242, - 3.839996802564, 3.84951355844971, 3.84228360113373, 3.82457408008348, - 3.83730050014674, 3.81602439649907, 3.79952375106963, 3.81216850691302, - 3.83475040413067, 3.84515172504442, 3.85524175045754, 3.85898334507543, - 3.86098785446156, 3.85897089801443, 3.86084133508315, 3.85946975293424, - 3.84228359857435, 3.82457407668573, 3.80871339914924, 3.82041520050351, - 3.83999679982684, 3.84951355646953, 3.86084133506967, 3.85897089799313, - 3.86098785445721, 3.8589833450892, 3.85524175050246, 3.85946975295874, - 3.83475040424791, 3.81216850705744, 3.79952375129251, 3.81602439672981, - 3.83730050032849, 3.84515172514695, 3.82457408051469, 3.84228360143838, - 3.84951355865041, 3.8399968028122, 3.82041520452226, 3.80871340383006, - 3.86084133539843, 3.85897089899106, 3.86098785465421, 3.85898334408679, - 3.8552417488472, 3.85946975207691, 3.83475040068896, 3.81216850212871, - 3.79952374582553, 3.81602439205011, 3.83730049695183, 3.84515172235639, - 3.82457407647112, 3.84228359847614, 3.84951355633341, 3.83999679965052, - 3.8204152001564, 3.80871339890026, 3.4648860671801, 3.4992446466888, - 3.57603284364376, 3.60145793367096, 3.57210179959393, 3.49454780394676, - 3.57210179985685, 3.60145793414029, 3.57603284441111, 3.49924464746755, - 3.4648860677802, 3.49454780421399, 3.2520792870555, 3.20616414912086, - 3.2387419765059, 3.34224676063611, 3.38203645550112, 3.35389277464523, - 3.08082659980027, 3.04241172894884, 2.9206912245272, 2.86476251997308, - 2.91050122114396, 3.03341253869673, 2.91050122079906, 2.86476251926956, - 2.92069122352958, 3.04241172788238, 3.08082659914702, 3.033412538331, - 3.35389277309147, 3.38203645440123, 3.34224675965113, 3.23874197560188, - 3.206164147754, 3.25207928564178, 2.28022859870545, 2.40265427033085, - 2.4295968774201, 2.34121441165764, 2.2118869666104, 2.17953696186471, - 1.98494059668979, 1.88375468687305, 1.75116084588048, 1.70415169724174, - 1.81775134649606, 1.94602812016069, 1.54672375502529, 1.41889854029187, - 1.41889854105772, 1.54672375559485, 1.66983704566626, 1.66983704501851, - 1.8177513481593, 1.70415169892245, 1.75116084857611, 1.88375468933316, - 1.9849405991179, 1.94602812170514, 2.21188696886946, 2.3412144138293, - 2.42959687955111, 2.40265427179645, 2.28022860012117, 2.17953696334986, - 2.48250524155944, 2.58069897831205, 2.58069897796797, 2.48250524096618, - 2.37040345532657, 2.37040345574696, 1.70415171207159, 1.81775135925391, - 1.94602813482244, 1.98494061211482, 1.88375470439201, 1.75116086138526, - 1.41889854829708, 1.54672376170737, 1.66983705336537, 1.66983705456452, - 1.54672376632688, 1.41889854955953, 1.81775135280092, 1.70415170224912, - 1.75116085007806, 1.88375469050174, 1.98494060130122, 1.94602812567358, - 2.2118869716462, 2.34121441649983, 2.42959688285938, 2.40265427689687, - 2.28022860569235, 2.17953696790764, 2.4825052492982, 2.5806989870123, - 2.58069898831851, 2.4825052523831, 2.37040346556608, 2.37040346431457, - 2.34121442840181, 2.21188698191546, 2.17953697632402, 2.28022861154443, - 2.40265428451121, 2.42959689251844, 1.81775136502913, 1.70415171757556, - 1.75116086502996, 1.88375470809254, 1.98494061622916, 1.94602814066237, - 2.21188698622573, 2.34121443274437, 2.4295968968027, 2.40265429031028, - 2.28022861731359, 2.17953698224677, 2.58069899496667, 2.48250525726598, - 2.3704034728221, 2.37040347257654, 2.48250525939679, 2.58069899482802, - 2.28022861555448, 2.4026542861926, 2.4295968931892, 2.3412144268598, - 2.21188698258817, 2.17953697788018, 1.98494061271456, 1.88375470218215, - 1.75116086179774, 1.70415171324896, 1.8177513634387, 1.94602813621702, - 1.41889855692285, 1.54672377371729, 1.66983706215624, 1.66983706264065, - 1.54672377126065, 1.41889855743696, 1.54672375422311, 1.41889853979863, - 1.41889853943904, 1.54672375443117, 1.6698370444596, 1.66983704465869, - 1.81775134660655, 1.70415169718831, 1.75116084605761, 1.88375468713567, - 1.98494059721597, 1.94602812044671, 2.21188696754454, 2.34121441266494, - 2.42959687876722, 2.4026542714854, 2.28022859968374, 2.17953696256324, - 2.48250524207868, 2.58069897922181, 2.58069897921054, 2.48250524198014, - 2.37040345608139, 2.37040345611661, 2.28022859950511, 2.40265427129882, - 2.42959687854649, 2.34121441254737, 2.21188696735763, 2.17953696233429, - 1.98494059721609, 1.88375468720816, 1.75116084642749, 1.70415169707677, - 1.81775134667405, 1.94602812022711, 3.85897089845485, 3.86084133519797, - 3.8594697525296, 3.8552417497379, 3.85898334455374, 3.86098785454272, - 3.84228359963091, 3.82457407770568, 3.80871340048796, 3.82041520132951, - 3.83999680072582, 3.84951355708345, 3.83730049793289, 3.81602439313399, - 3.79952374711499, 3.81216850309196, 3.83475040154957, 3.84515172301957, - 3.85524174925527, 3.85898334428236, 3.86098785460524, 3.85897089874415, - 3.86084133529588, 3.85946975229149, 3.84228360050848, 3.82457407890995, - 3.808713401902, 3.82041520249319, 3.83999680161838, 3.84951355775291, - 3.81216850420119, 3.83475040251148, 3.84515172372913, 3.8373004988564, - 3.81602439426909, 3.79952374850035, 3.8608413352336, 3.85897089853854, - 3.86098785455535, 3.85898334449836, 3.85524174961272, 3.85946975244585, - 3.83475040220811, 3.81216850381735, 3.79952374798096, 3.81602439371894, - 3.83730049842566, 3.84515172343446, 3.82457407797039, 3.8422835997821, - 3.8495135572642, 3.8399968010498, 3.82041520176199, 3.80871340093741, - 3.8608413352106, 3.85897089845091, 3.86098785453913, 3.85898334461991, - 3.85524174976196, 3.85946975252995, 3.83475040255377, 3.81216850450131, - 3.79952374854541, 3.81602439422395, 3.83730049868561, 3.84515172371925, - 3.82457407823984, 3.84228359998145, 3.84951355742012, 3.83999680125938, - 3.8204152021239, 3.80871340125699, 3.04241173682453, 3.08082660801718, - 3.0334125478374, 2.9105012306686, 2.86476252917546, 2.92069123275721, - 3.35389278008578, 3.38203646125342, 3.34224676714461, 3.2387419834683, - 3.20616415572886, 3.25207929293555, 3.46488607403731, 3.49924465294645, - 3.57603284944596, 3.60145793920913, 3.57210180569235, 3.49454781055657, - 3.5721018061827, 3.60145794013974, 3.57603285089434, 3.49924465455745, - 3.46488607518046, 3.49454781110009, 3.38203646367467, 3.35389278326265, - 3.25207929631784, 3.20616415863301, 3.2387419855312, 3.34224676909527, - 2.86476253095995, 2.9105012315398, 3.03341254865552, 3.08082660957038, - 3.04241173925897, 2.92069123535956, 2.28022860654879, 2.40265427847283, - 2.42959688613419, 2.34121442127905, 2.21188697610917, 2.17953697069598, - 1.9849406070211, 1.88375469823444, 1.75116085710719, 1.70415170756799, - 1.81775135569698, 1.94602812971534, 1.54672376368563, 1.41889854835936, - 1.41889854747541, 1.54672376143889, 1.66983705198027, 1.66983705266572, - 1.81775135254055, 1.70415170279434, 1.75116085122506, 1.88375469162434, - 1.98494060192661, 1.94602812550664, 2.21188697166056, 2.34121441610718, - 2.4295968822255, 2.40265427515554, 2.28022860418704, 2.17953696695939, - 2.48250524607293, 2.58069898330845, 2.58069898360736, 2.48250524752886, - 2.37040346146523, 2.37040346111974, 3.08082660167768, 3.04241173059116, - 2.92069122656976, 2.86476252206777, 2.91050122343422, 3.03341254063995, - 2.91050122330072, 2.86476252177827, 2.9206912261252, 3.04241173027962, - 3.080826601482, 3.03341254054833, 3.2061641500075, 3.25207928778573, - 3.35389277511681, 3.38203645632499, 3.34224676160499, 3.23874197774547, - 3.46488606879207, 3.49924464832333, 3.57603284505016, 3.60145793499066, - 3.5721018008835, 3.49454780543662, 3.5721018009682, 3.60145793511002, - 3.57603284532914, 3.49924464850608, 3.4648860689349, 3.49454780548833, - 3.38203645662013, 3.35389277555892, 3.25207928829333, 3.20616415034956, - 3.23874197798444, 3.34224676181183, 3.81602439252289, 3.8373004972227, - 3.84515172264257, 3.83475040104545, 3.81216850268673, 3.79952374635626, - 3.82457407668573, 3.84228359857435, 3.84951355646953, 3.83999679982684, - 3.82041520050351, 3.80871339914924, 3.86084133508315, 3.85897089801443, - 3.86098785446156, 3.85898334507543, 3.85524175045754, 3.85946975293424, - 3.83475040413067, 3.81216850691302, 3.79952375106963, 3.81602439649907, - 3.83730050014675, 3.84515172504442, 3.82041520419242, 3.80871340338233, - 3.82457408008348, 3.84228360113373, 3.84951355844971, 3.839996802564, - 3.85898334420375, 3.85524174902513, 3.85946975217001, 3.86084133536623, - 3.85897089887936, 3.86098785463037, 2.40265427033085, 2.28022859870546, - 2.17953696186473, 2.21188696661041, 2.34121441165766, 2.4295968774201, - 2.58069897831205, 2.48250524155943, 2.37040345574696, 2.37040345532657, - 2.48250524096618, 2.58069897796796, 2.28022860012117, 2.40265427179644, - 2.42959687955111, 2.34121441382928, 2.21188696886945, 2.17953696334984, - 1.9849405991179, 1.88375468933315, 1.75116084857611, 1.70415169892243, - 1.81775134815931, 1.94602812170512, 1.41889854029187, 1.54672375502529, - 1.6698370450185, 1.66983704566626, 1.54672375559484, 1.41889854105773, - 1.88375468687307, 1.98494059668979, 1.94602812016069, 1.81775134649606, - 1.70415169724175, 1.75116084588049, 3.08082659914702, 3.04241172788239, - 2.92069122352959, 2.86476251926958, 2.91050122079907, 3.03341253833101, - 2.91050122114398, 2.86476251997311, 2.92069122452722, 3.04241172894886, - 3.08082659980029, 3.03341253869675, 3.20616414912087, 3.25207928705551, - 3.35389277464524, 3.38203645550113, 3.34224676063612, 3.23874197650591, - 3.4648860677802, 3.49924464746755, 3.57603284441111, 3.60145793414029, - 3.57210179985686, 3.49454780421399, 3.57210179959394, 3.60145793367097, - 3.57603284364376, 3.4992446466888, 3.4648860671801, 3.49454780394675, - 3.38203645440123, 3.35389277309147, 3.25207928564178, 3.20616414775401, - 3.23874197560188, 3.34224675965113, 3.83730049695183, 3.81602439205011, - 3.79952374582553, 3.81216850212871, 3.83475040068896, 3.84515172235639, - 3.8552417488472, 3.85898334408679, 3.86098785465421, 3.85897089899106, - 3.86084133539843, 3.85946975207691, 3.84228360143838, 3.82457408051469, - 3.80871340383006, 3.82041520452226, 3.8399968028122, 3.84951355865041, - 3.83730050032849, 3.81602439672981, 3.79952375129251, 3.81216850705743, - 3.83475040424791, 3.84515172514695, 3.85524175050246, 3.8589833450892, - 3.86098785445721, 3.85897089799313, 3.86084133506967, 3.85946975295874, - 3.84228359847614, 3.82457407647112, 3.80871339890026, 3.8204152001564, - 3.83999679965052, 3.84951355633341, 3.04241174664212, 3.08082661685684, - 3.03341255683306, 2.91050123934182, 2.8647625390975, 2.92069124230874, - 3.20616416504156, 3.25207930188615, 3.35389278857596, 3.38203646925399, - 3.34224677531644, 3.23874199205821, 3.46488608097362, 3.49924465984842, - 3.5760328554895, 3.60145794491511, 3.57210181135635, 3.49454781701208, - 3.57210181127168, 3.60145794473248, 3.57603285522568, 3.49924465945314, - 3.46488608071886, 3.49454781688324, 3.25207930071066, 3.20616416394185, - 3.23874199136287, 3.3422467747452, 3.38203646853347, 3.35389278760403, - 2.86476253812043, 2.91050123895668, 3.0334125564585, 3.08082661625989, - 3.04241174551892, 2.92069124115386, 2.28022861731358, 2.40265429031028, - 2.4295968968027, 2.34121443274438, 2.21188698622572, 2.17953698224677, - 1.98494061622916, 1.88375470809255, 1.75116086502996, 1.70415171757556, - 1.81775136502913, 1.94602814066238, 1.5467237737173, 1.41889855692284, - 1.41889855743697, 1.54672377126063, 1.66983706264066, 1.66983706215624, - 1.81775136343869, 1.70415171324895, 1.75116086179774, 1.88375470218214, - 1.98494061271454, 1.94602813621699, 2.21188698258816, 2.34121442685979, - 2.42959689318918, 2.40265428619257, 2.28022861555447, 2.17953697788015, - 2.48250525726596, 2.58069899496666, 2.580698994828, 2.48250525939677, - 2.37040347257651, 2.37040347282208, 3.83730049953695, 3.81602439552504, - 3.79952374979441, 3.8121685055086, 3.83475040322909, 3.84515172438773, - 3.85524175001268, 3.85898334474675, 3.86098785451683, 3.85897089828832, - 3.86084133515291, 3.85946975270758, 3.8422835992181, 3.82457407735823, - 3.80871339993605, 3.82041520095658, 3.8399968002872, 3.8495135568304, - 3.83730049748391, 3.81602439274038, 3.79952374653887, 3.81216850266742, - 3.83475040109803, 3.84515172273185, 3.85524174900921, 3.85898334416065, - 3.86098785463868, 3.85897089888909, 3.86084133535606, 3.85946975218227, - 3.84228360104085, 3.82457407991773, 3.80871340299439, 3.82041520364015, - 3.83999680222929, 3.84951355826986, 3.04241173024201, 3.08082660099625, - 3.03341254061537, 2.91050122232067, 2.86476252123369, 2.92069122521197, - 3.35389277617434, 3.38203645687034, 3.34224676258644, 3.23874197779162, - 3.2061641505101, 3.25207928805274, 3.46488607017294, 3.49924464917407, - 3.57603284670063, 3.60145793609385, 3.57210180252023, 3.49454780628909, - 3.57210180267431, 3.60145793638611, 3.57603284716961, 3.4992446496989, - 3.46488607056531, 3.49454780646093, 3.38203645769644, 3.3538927774706, - 3.25207928927346, 3.206164151828, 3.23874197850084, 3.34224676335215, - 2.86476252219831, 2.91050122267359, 3.0334125410174, 3.08082660156323, - 3.04241173145599, 2.92069122628746, 2.40265427129883, 2.2802285995051, - 2.1795369623343, 2.21188696735762, 2.34121441254738, 2.42959687854648, - 2.58069897922181, 2.48250524207867, 2.37040345611661, 2.37040345608139, - 2.48250524198015, 2.58069897921053, 2.28022859968375, 2.40265427148541, - 2.42959687876723, 2.34121441266495, 2.21188696754455, 2.17953696256324, - 1.98494059721598, 1.88375468713568, 1.75116084605762, 1.70415169718831, - 1.81775134660656, 1.94602812044671, 1.41889853979862, 1.54672375422312, - 1.66983704465869, 1.6698370444596, 1.54672375443116, 1.41889853943904, - 1.88375468720816, 1.98494059721608, 1.94602812022712, 1.81775134667405, - 1.70415169707678, 1.75116084642747, 3.08082660099623, 3.04241173024201, - 2.92069122521196, 2.8647625212337, 2.91050122232066, 3.03341254061537, - 2.91050122267358, 2.86476252219832, 2.92069122628745, 3.042411731456, - 3.08082660156322, 3.0334125410174, 3.3538927774706, 3.38203645769642, - 3.34224676335215, 3.23874197850083, 3.206164151828, 3.25207928927345, - 3.46488607056531, 3.49924464969888, 3.57603284716962, 3.6014579363861, - 3.57210180267431, 3.49454780646091, 3.57210180252023, 3.60145793609383, - 3.57603284670063, 3.49924464917406, 3.46488607017293, 3.49454780628908, - 3.38203645687032, 3.35389277617434, 3.25207928805273, 3.2061641505101, - 3.2387419777916, 3.34224676258644, 3.82457407735823, 3.8422835992181, - 3.8495135568304, 3.8399968002872, 3.82041520095658, 3.80871339993605, - 3.86084133515291, 3.85897089828832, 3.86098785451683, 3.85898334474675, - 3.85524175001267, 3.85946975270758, 3.83475040322909, 3.8121685055086, - 3.79952374979441, 3.81602439552503, 3.83730049953695, 3.84515172438773, - 3.82457407991773, 3.84228360104086, 3.84951355826986, 3.8399968022293, - 3.82041520364015, 3.80871340299439, 3.85524174900921, 3.85946975218227, - 3.86084133535606, 3.85897089888909, 3.86098785463868, 3.85898334416065, - 3.83475040109803, 3.81216850266742, 3.79952374653887, 3.81602439274038, - 3.83730049748391, 3.84515172273185, 1.81775135925393, 1.7041517120716, - 1.75116086138526, 1.88375470439201, 1.98494061211483, 1.94602813482244, - 2.21188698191547, 2.34121442840182, 2.42959689251846, 2.40265428451122, - 2.28022861154444, 2.17953697632403, 2.58069898701232, 2.48250524929822, - 2.37040346431457, 2.37040346556609, 2.48250525238311, 2.58069898831853, - 2.28022860569234, 2.40265427689689, 2.42959688285938, 2.34121441649984, - 2.21188697164619, 2.17953696790764, 1.98494060130121, 1.88375469050176, - 1.75116085007805, 1.70415170224913, 1.81775135280091, 1.94602812567358, - 1.5467237617074, 1.41889854829708, 1.41889854955954, 1.54672376632688, - 1.66983705456452, 1.66983705336537, 3.46488608071885, 3.49924465945314, - 3.57603285522568, 3.60145794473248, 3.57210181127168, 3.49454781688323, - 3.57210181135634, 3.60145794491511, 3.57603285548949, 3.49924465984842, - 3.46488608097362, 3.49454781701208, 3.25207930188615, 3.20616416504156, - 3.23874199205821, 3.34224677531644, 3.38203646925399, 3.35389278857596, - 3.08082661685684, 3.04241174664212, 2.92069124230873, 2.86476253909749, - 2.91050123934182, 3.03341255683306, 2.91050123895668, 2.86476253812042, - 2.92069124115385, 3.04241174551892, 3.08082661625989, 3.0334125564585, - 3.20616416394184, 3.25207930071066, 3.35389278760403, 3.38203646853347, - 3.34224677474519, 3.23874199136287, 3.17505517819785, 3.16097552492228, - 3.05899736283131, 2.99386884005565, 3.00977836813594, 3.11994727779011, - 2.97060165769012, 2.90894311915608, 2.94084144324995, 3.06015736421514, - 3.1132519171817, 3.08618979269084, 3.58371794822061, 3.61624399712336, - 3.59477675154682, 3.52186831530198, 3.48385686711648, 3.50846685621332, - 3.60854784965156, 3.64295339381216, 3.62386338594055, 3.55882608789277, - 3.51867229952132, 3.53891420750262, 3.68589812473734, 3.70431753793426, - 3.74976895373651, 3.77230076655113, 3.75703001910139, 3.71440072749805, - 3.85215761997655, 3.84461773387907, 3.82904399367736, 3.81445584391205, - 3.82756485329104, 3.8438449880284, 3.79191280344149, 3.77128289386491, - 3.74057496637648, 3.71478050850439, 3.74057496659819, 3.77128289404559, - 3.82904399357776, 3.84461773381541, 3.85215761988125, 3.84384498783089, - 3.8275648529848, 3.81445584367382, 3.74976895195735, 3.70431753581031, - 3.68589812270217, 3.714400725886, 3.75703001776383, 3.77230076515159, - 3.60854784747611, 3.53891420513236, 3.51867229735203, 3.55882608622736, - 3.62386338439428, 3.64295339210458, 3.59477674871981, 3.61624399419415, - 3.58371794474004, 3.50846685243702, 3.48385686344634, 3.52186831222077, - 2.94084143817571, 2.90894311438688, 2.97060165380045, 3.08618978857682, - 3.11325191276977, 3.06015735901039, 3.00977836548108, 2.99386883786249, - 3.05899736141099, 3.1609755230836, 3.17505517590482, 3.11994727480999, - 2.82990733805891, 2.75803570313654, 2.63612385724924, 2.61366767714787, - 2.69774163112156, 2.81113256117699, 2.75803570716211, 2.829907341092, - 2.811132563732, 2.69774163294624, 2.61366768007544, 2.63612386069404, - 3.05899735743508, 3.16097551930725, 3.17505517241248, 3.11994727121531, - 3.00977836163036, 2.99386883382839, 2.82990733386299, 2.75803569910261, - 2.63612385273211, 2.61366767259434, 2.69774162636306, 2.81113255688698, - 2.63612385473025, 2.75803570117254, 2.8299073356907, 2.81113255834232, - 2.69774162755968, 2.61366767405634, 3.0097783632334, 3.11994727309666, - 3.17505517374158, 3.1609755206402, 3.05899735828153, 2.99386883525182, - 2.97060165232267, 2.90894311323227, 2.94084143729868, 3.06015735837716, - 3.11325191189141, 3.08618978747851, 3.58371794419843, 3.61624399366165, - 3.59477674782223, 3.52186831151937, 3.48385686281036, 3.50846685204722, - 3.60854784613328, 3.64295339082796, 3.62386338255703, 3.55882608463346, - 3.51867229580091, 3.53891420401672, 3.68589812153703, 3.70431753511233, - 3.74976895097958, 3.77230076432702, 3.75703001641713, 3.71440072485043, - 3.8521576191201, 3.84461773254023, 3.82904399225209, 3.81445584187369, - 3.82756485180208, 3.84384498665062, 3.79191280163363, 3.77128289151291, - 3.74057496432901, 3.71478050575604, 3.74057496440724, 3.77128289155391, - 3.82904399221832, 3.84461773252718, 3.85215761908784, 3.84384498660469, - 3.82756485169123, 3.8144558418203, 3.74976895027194, 3.70431753411461, - 3.68589812065336, 3.71440072405832, 3.75703001590384, 3.77230076368046, - 3.60854784508713, 3.53891420273524, 3.51867229462836, 3.55882608373335, - 3.62386338186617, 3.64295338994111, 3.59477674652184, 3.61624399231187, - 3.58371794277901, 3.5084668503841, 3.48385686111259, 3.52186830997292, - 2.90894311085056, 2.97060165015118, 3.08618978522416, 3.11325190964041, - 3.06015735595416, 2.94084143479796, 0.829324749822268, 0.719686353737887, - 0.829324750169392, 0.957414201276656, 1.04507987304047, - 0.957414200969615, 1.27627149123162, 1.40803971968493, 1.51358653750022, - 1.4563662285434, 1.32613276581893, 1.20587112141495, 1.5135865384577, - 1.40803972076972, 1.27627149303911, 1.20587112207766, 1.32613276662738, - 1.45636622869928, 1.04507987600702, 0.957414204893307, 0.829324754594157, - 0.719686356648459, 0.829324752096637, 0.957414202494864, - 1.27627149478415, 1.40803972236866, 1.51358654136619, 1.45636623337298, - 1.3261327717817, 1.20587112590036, 1.51358655314004, 1.40803973787705, - 1.27627150701867, 1.20587113710113, 1.3261327789213, 1.45636624385303, - 1.04507988867353, 0.957414219263204, 0.829324764975279, - 0.719686369895612, 0.829324762817598, 0.957414216989921, 1.2762715086727, - 1.40803973958239, 1.51358655598572, 1.45636624871029, 1.3261327836562, - 1.20587114092256, 1.51358655319684, 1.40803973480565, 1.27627150647983, - 1.20587113747749, 1.32613278250847, 1.4563662447156, 1.04507988721126, - 0.957414214549234, 0.829324762185518, 0.719686368197244, 0.8293247645036, - 0.957414216583879, 1.27627150457475, 1.40803973320339, 1.51358654975237, - 1.45636624000749, 1.32613277690619, 1.20587113394636, 1.51358654290189, - 1.40803972445672, 1.27627149654387, 1.20587112747997, 1.3261327725023, - 1.45636623451286, 1.04507987780354, 0.957414204615115, 0.8293247537153, - 0.71968635851059, 0.829324755789629, 0.957414206727861, 1.27627149516361, - 1.4080397231698, 1.5135865406883, 1.45636623078723, 1.3261327684941, - 1.20587112426236, 1.40803971976829, 1.27627149136335, 1.20587112180345, - 1.32613276625679, 1.45636622883327, 1.51358653755931, 3.17505517590482, - 3.16097552308359, 3.05899736141099, 2.99386883786248, 3.00977836548107, - 3.11994727480998, 2.97060165380045, 2.90894311438687, 2.94084143817571, - 3.06015735901038, 3.11325191276977, 3.08618978857681, 3.58371794474004, - 3.61624399419416, 3.59477674871981, 3.52186831222077, 3.48385686344634, - 3.50846685243703, 3.6085478474761, 3.64295339210458, 3.62386338439427, - 3.55882608622737, 3.51867229735202, 3.53891420513237, 3.68589812270216, - 3.70431753581031, 3.74976895195734, 3.77230076515159, 3.75703001776383, - 3.714400725886, 3.85215761988125, 3.8446177338154, 3.82904399357776, - 3.81445584367381, 3.82756485298481, 3.84384498783089, 3.7919128034415, - 3.77128289404559, 3.74057496659821, 3.71478050850439, 3.74057496637649, - 3.77128289386491, 3.82904399367736, 3.84461773387906, 3.85215761997655, - 3.8438449880284, 3.82756485329105, 3.81445584391204, 3.74976895373651, - 3.70431753793427, 3.68589812473734, 3.71440072749807, 3.75703001910139, - 3.77230076655114, 3.60854784965156, 3.53891420750263, 3.51867229952133, - 3.55882608789279, 3.62386338594055, 3.64295339381217, 3.59477675154682, - 3.61624399712337, 3.58371794822061, 3.50846685621333, 3.48385686711648, - 3.52186831530199, 2.94084144324996, 2.90894311915608, 2.97060165769014, - 3.08618979269085, 3.11325191718171, 3.06015736421513, 3.00977836813596, - 2.99386884005566, 3.05899736283132, 3.16097552492229, 3.17505517819786, - 3.11994727779012, 2.82990734109201, 2.75803570716211, 2.63612386069404, - 2.61366768007543, 2.69774163294624, 2.81113256373201, 2.75803570313653, - 2.82990733805891, 2.81113256117699, 2.69774163112156, 2.61366767714787, - 2.63612385724924, 3.74057496440724, 3.71478050575604, 3.74057496432901, - 3.77128289151291, 3.79191280163363, 3.7712828915539, 3.82904399225209, - 3.84461773254023, 3.8521576191201, 3.84384498665062, 3.82756485180208, - 3.81445584187369, 3.74976895097957, 3.70431753511233, 3.68589812153703, - 3.71440072485042, 3.75703001641713, 3.77230076432701, 3.60854784613328, - 3.53891420401671, 3.5186722958009, 3.55882608463346, 3.62386338255703, - 3.64295339082796, 3.59477674782222, 3.61624399366165, 3.58371794419843, - 3.50846685204721, 3.48385686281036, 3.52186831151937, 2.94084143729867, - 2.90894311323226, 2.97060165232266, 3.0861897874785, 3.11325191189141, - 3.06015735837715, 3.00977836323339, 2.99386883525181, 3.05899735828152, - 3.1609755206402, 3.17505517374158, 3.11994727309664, 2.82990733569069, - 2.75803570117252, 2.63612385473025, 2.61366767405633, 2.69774162755968, - 2.8111325583423, 2.6361238527321, 2.75803569910262, 2.829907333863, - 2.811132556887, 2.69774162636306, 2.61366767259434, 3.00977836163036, - 3.11994727121532, 3.17505517241248, 3.16097551930725, 3.05899735743508, - 2.99386883382839, 2.97060165015119, 2.90894311085058, 2.94084143479797, - 3.06015735595416, 3.11325190964042, 3.08618978522417, 3.583717942779, - 3.61624399231187, 3.59477674652184, 3.52186830997292, 3.4838568611126, - 3.5084668503841, 3.60854784508713, 3.64295338994111, 3.62386338186617, - 3.55882608373334, 3.51867229462835, 3.53891420273523, 3.68589812065336, - 3.7043175341146, 3.74976895027195, 3.77230076368046, 3.75703001590384, - 3.71440072405832, 3.84461773252718, 3.82904399221832, 3.8144558418203, - 3.82756485169123, 3.84384498660469, 3.85215761908784, 3.51867230681858, - 3.55882609482331, 3.62386339196214, 3.64295339969012, 3.60854785582946, - 3.53891421460591, 3.59477675781379, 3.61624400319028, 3.58371795445638, - 3.50846686334877, 3.48385687456329, 3.52186832248982, 2.94084145333509, - 2.90894313019809, 2.97060166803225, 3.08618980314812, 3.11325192691397, - 3.06015737450286, 3.00977837854318, 2.99386885102277, 3.05899737318336, - 3.16097553510326, 3.17505518797796, 3.11994728818731, 2.82990735187877, - 2.75803571871458, 2.63612387157793, 2.61366769189825, 2.69774164402701, - 2.81113257525686, 2.63612386811255, 2.75803571373755, 2.82990734865515, - 2.81113257199718, 2.69774164234379, 2.6136676881632, 3.00977837562289, - 3.11994728458699, 3.17505518540039, 3.16097553290413, 3.05899737171443, - 2.99386884828909, 2.97060166362898, 2.90894312436941, 2.94084144754962, - 3.06015736820745, 3.11325192172779, 3.08618979814766, 3.58371795056125, - 3.61624399975918, 3.59477675467844, 3.52186831884396, 3.4838568703685, - 3.50846685883469, 3.60854785345252, 3.6429533977543, 3.62386339029672, - 3.55882609292684, 3.51867230442272, 3.53891421185193, 3.68589812755707, - 3.7043175403825, 3.74976895570274, 3.77230076861619, 3.75703002144123, - 3.71440073044731, 3.85215762093028, 3.84461773513492, 3.82904399568355, - 3.81445584604517, 3.82756485512302, 3.84384498917669, 3.79191280667193, - 3.77128289754252, 3.74057497097852, 3.71478051314172, 3.74057497073431, - 3.77128289734671, 3.82904399579575, 3.84461773521061, 3.85215762104181, - 3.84384498940757, 3.82756485546706, 3.81445584631225, 3.70431754280801, - 3.68589812975883, 3.71440073226354, 3.75703002287439, 3.77230077018087, - 3.7497689576034, 3.6238633838419, 3.55882608566565, 3.51867229680122, - 3.53891420454323, 3.60854784690197, 3.64295339161911, 3.68589812208833, - 3.70431753530002, 3.74976895133279, 3.77230076470937, 3.75703001715803, - 3.7144007254256, 3.85215761968578, 3.84461773349495, 3.82904399331002, - 3.8144558432207, 3.82756485267547, 3.84384498745929, 3.79191280314032, - 3.7712828936652, 3.74057496622281, 3.71478050808158, 3.74057496601206, - 3.77128289341573, 3.82904399344128, 3.84461773361157, 3.85215761983193, - 3.84384498780549, 3.82756485306006, 3.81445584359317, 3.74976895293871, - 3.70431753660344, 3.68589812369588, 3.71440072652904, 3.75703001853086, - 3.77230076581546, 3.60854784835546, 3.53891420554641, 3.51867229795721, - 3.55882608649137, 3.62386338504842, 3.64295339261555, 3.59477674977981, - 3.61624399496551, 3.58371794595683, 3.50846685302161, 3.48385686437887, - 3.52186831273195, 2.94084143674032, 2.90894311305048, 2.97060165237489, - 3.08618978786781, 3.11325191197401, 3.06015735845159, 3.0097783640784, - 2.99386883625511, 3.05899735997717, 3.16097552205816, 3.17505517514038, - 3.1199472739597, 2.82990733591079, 2.75803570097618, 2.63612385431683, - 2.61366767409011, 2.69774162810981, 2.81113255888408, 2.63612385444689, - 2.75803570096113, 2.82990733585776, 2.81113255894476, 2.69774162815067, - 2.61366767421521, 3.00977836378975, 3.11994727369897, 3.17505517461689, - 3.16097552195971, 3.05899735965723, 2.99386883624721, 2.97060165209063, - 2.90894311271201, 2.94084143634347, 3.06015735763346, 3.11325191135367, - 3.08618978738398, 3.61624399356675, 3.59477674808602, 3.52186831154629, - 3.48385686272553, 3.50846685160897, 3.5837179439899, 3.51867229680123, - 3.55882608566566, 3.62386338384191, 3.64295339161911, 3.60854784690197, - 3.53891420454323, 3.59477674808602, 3.61624399356676, 3.58371794398991, - 3.50846685160897, 3.48385686272553, 3.52186831154629, 2.94084143634348, - 2.90894311271201, 2.97060165209063, 3.08618978738399, 3.11325191135367, - 3.06015735763348, 3.00977836378975, 2.99386883624721, 3.05899735965724, - 3.16097552195972, 3.17505517461689, 3.11994727369899, 2.82990733585777, - 2.75803570096114, 2.63612385444689, 2.61366767421522, 2.69774162815068, - 2.81113255894476, 2.63612385431684, 2.75803570097618, 2.82990733591078, - 2.81113255888409, 2.69774162810982, 2.61366767409012, 3.0097783640784, - 3.1199472739597, 3.17505517514038, 3.16097552205816, 3.05899735997718, - 2.99386883625512, 2.97060165237488, 2.90894311305048, 2.94084143674031, - 3.06015735845159, 3.113251911974, 3.08618978786781, 3.58371794595682, - 3.61624399496549, 3.59477674977981, 3.52186831273193, 3.48385686437887, - 3.5084668530216, 3.60854784835546, 3.64295339261554, 3.62386338504842, - 3.55882608649137, 3.51867229795721, 3.53891420554639, 3.68589812369588, - 3.70431753660344, 3.74976895293871, 3.77230076581545, 3.75703001853087, - 3.71440072652904, 3.85215761983192, 3.84461773361157, 3.82904399344129, - 3.81445584359317, 3.82756485306006, 3.84384498780549, 3.79191280314032, - 3.77128289341573, 3.74057496601206, 3.71478050808159, 3.74057496622281, - 3.7712828936652, 3.82904399331002, 3.84461773349495, 3.85215761968578, - 3.84384498745929, 3.82756485267547, 3.8144558432207, 3.70431753530002, - 3.68589812208834, 3.7144007254256, 3.75703001715803, 3.77230076470937, - 3.74976895133279, 0.829324750169394, 0.71968635373788, 0.829324749822264, - 0.957414200969637, 1.04507987304049, 0.957414201276675, 1.27627149136335, - 1.4080397197683, 1.51358653755931, 1.45636622883328, 1.3261327662568, - 1.20587112180346, 1.5135865406883, 1.40803972316978, 1.27627149516361, - 1.20587112426234, 1.3261327684941, 1.45636623078722, 1.04507987780355, - 0.957414206727851, 0.829324755789636, 0.719686358510584, - 0.829324753715309, 0.957414204615115, 1.27627149654387, 1.40803972445672, - 1.5135865429019, 1.45636623451286, 1.32613277250231, 1.20587112747998, - 1.51358654975237, 1.40803973320338, 1.27627150457475, 1.20587113394636, - 1.32613277690619, 1.45636624000748, 1.04507988721126, 0.957414216583866, - 0.829324764503594, 0.719686368197235, 0.829324762185524, - 0.957414214549229, 1.27627150647983, 1.40803973480564, 1.51358655319684, - 1.45636624471558, 1.32613278250847, 1.20587113747747, 1.51358655598572, - 1.40803973958239, 1.2762715086727, 1.20587114092257, 1.32613278365619, - 1.45636624871029, 1.04507988867352, 0.957414216989908, 0.829324762817598, - 0.719686369895601, 0.829324764975269, 0.957414219263202, - 1.27627150701867, 1.40803973787704, 1.51358655314004, 1.45636624385303, - 1.32613277892131, 1.20587113710114, 1.51358654136616, 1.40803972236866, - 1.27627149478413, 1.20587112590037, 1.3261327717817, 1.45636623337298, - 1.045079876007, 0.957414202494867, 0.829324752096628, 0.719686356648451, - 0.829324754594135, 0.957414204893297, 1.27627149303908, 1.40803972076972, - 1.51358653845769, 1.45636622869928, 1.32613276662738, 1.20587112207765, - 1.40803971968494, 1.27627149123163, 1.20587112141496, 1.32613276581896, - 1.4563662285434, 1.51358653750022, 3.51867230530465, 3.55882609310292, - 3.62386339067989, 3.64295339853009, 3.60854785484376, 3.53891421325143, - 3.59477675710285, 3.61624400257861, 3.58371795398677, 3.50846686266358, - 3.48385687373568, 3.52186832148399, 2.94084145141462, 2.90894312732319, - 2.97060166536041, 3.0861898005471, 3.11325192510552, 3.06015737263463, - 3.00977837515217, 2.99386884646424, 3.05899736912919, 3.16097553114405, - 3.1750551849751, 3.11994728488167, 2.82990734810756, 2.75803571461202, - 2.63612386749075, 2.61366768638992, 2.69774163895515, 2.81113257017222, - 2.63612385851353, 2.75803570508642, 2.82990733988544, 2.81113256429108, - 2.69774163377649, 2.61366767963183, 3.00977836776533, 3.11994727781453, - 3.17505517852964, 3.16097552690989, 3.05899736478825, 2.99386884143827, - 2.97060165508268, 2.90894311601725, 2.94084143858907, 3.06015736050123, - 3.11325191378591, 3.08618979079264, 3.58371794676065, 3.61624399590161, - 3.59477675095128, 3.5218683140611, 3.48385686560797, 3.50846685392672, - 3.60854784994205, 3.64295339436361, 3.62386338712135, 3.55882608885917, - 3.5186723001745, 3.53891420735824, 3.68589812482293, 3.70431753752687, - 3.74976895372371, 3.77230076667804, 3.7570300196041, 3.71440072778249, - 3.85215762056045, 3.84461773471831, 3.82904399494469, 3.81445584521316, - 3.82756485425672, 3.84384498871125, 3.79191280555792, 3.771282896499, - 3.74057496950739, 3.71478051155035, 3.7405749689668, 3.77128289612978, - 3.82904399516701, 3.84461773483012, 3.85215762076234, 3.84384498906083, - 3.82756485493774, 3.81445584566869, 3.70431754228365, 3.68589812913556, - 3.71440073152898, 3.75703002232892, 3.77230076974821, 3.74976895722169, - 3.62386339067989, 3.55882609310293, 3.51867230530466, 3.53891421325144, - 3.60854785484375, 3.64295339853008, 3.68589812913556, 3.70431754228365, - 3.74976895722168, 3.77230076974821, 3.75703002232892, 3.71440073152898, - 3.85215762076234, 3.84461773483012, 3.82904399516701, 3.8144558456687, - 3.82756485493774, 3.84384498906083, 3.79191280555792, 3.77128289612979, - 3.7405749689668, 3.71478051155035, 3.74057496950739, 3.77128289649901, - 3.8290439949447, 3.84461773471831, 3.85215762056045, 3.84384498871125, - 3.82756485425673, 3.81445584521317, 3.74976895372371, 3.70431753752687, - 3.68589812482294, 3.71440072778249, 3.7570300196041, 3.77230076667804, - 3.60854784994206, 3.53891420735824, 3.51867230017451, 3.55882608885918, - 3.62386338712136, 3.64295339436362, 3.59477675095128, 3.61624399590161, - 3.58371794676065, 3.50846685392671, 3.48385686560797, 3.5218683140611, - 2.94084143858907, 2.90894311601726, 2.97060165508268, 3.08618979079265, - 3.11325191378591, 3.06015736050124, 3.00977836776534, 2.99386884143828, - 3.05899736478827, 3.1609755269099, 3.17505517852965, 3.11994727781455, - 2.82990733988545, 2.75803570508643, 2.63612385851352, 2.61366767963184, - 2.69774163377649, 2.8111325642911, 2.63612386749076, 2.75803571461203, - 2.82990734810755, 2.81113257017222, 2.69774163895516, 2.61366768638993, - 3.00977837515218, 3.11994728488168, 3.1750551849751, 3.16097553114405, - 3.0589973691292, 2.99386884646425, 2.97060166536041, 2.90894312732319, - 2.94084145141462, 3.06015737263463, 3.11325192510551, 3.0861898005471, - 3.61624400257861, 3.59477675710285, 3.52186832148399, 3.48385687373568, - 3.50846686266358, 3.58371795398677, 3.62386339196214, 3.55882609482331, - 3.51867230681857, 3.5389142146059, 3.60854785582946, 3.64295339969012, - 3.68589812975883, 3.70431754280801, 3.7497689576034, 3.77230077018087, - 3.75703002287439, 3.71440073226354, 3.85215762104181, 3.84461773521062, - 3.82904399579574, 3.81445584631225, 3.82756485546705, 3.84384498940757, - 3.79191280667193, 3.77128289734671, 3.74057497073431, 3.71478051314172, - 3.74057497097852, 3.77128289754252, 3.82904399568355, 3.84461773513492, - 3.85215762093028, 3.84384498917669, 3.82756485512302, 3.81445584604517, - 3.74976895570274, 3.7043175403825, 3.68589812755708, 3.71440073044731, - 3.75703002144123, 3.77230076861619, 3.60854785345253, 3.53891421185193, - 3.51867230442274, 3.55882609292684, 3.62386339029672, 3.6429533977543, - 3.59477675467844, 3.61624399975918, 3.58371795056126, 3.50846685883469, - 3.48385687036851, 3.52186831884397, 2.94084144754963, 2.90894312436943, - 2.97060166362898, 3.08618979814767, 3.1132519217278, 3.06015736820747, - 3.0097783756229, 2.9938688482891, 3.05899737171444, 3.16097553290414, - 3.17505518540039, 3.11994728458699, 2.82990734865515, 2.75803571373757, - 2.63612386811256, 2.61366768816322, 2.6977416423438, 2.8111325719972, - 2.63612387157793, 2.75803571871459, 2.82990735187878, 2.81113257525687, - 2.69774164402702, 2.61366769189824, 3.00977837854319, 3.11994728818731, - 3.17505518797796, 3.16097553510325, 3.05899737318337, 2.99386885102277, - 2.97060166803226, 2.90894313019808, 2.94084145333509, 3.06015737450286, - 3.11325192691397, 3.08618980314813, 3.61624400319029, 3.59477675781379, - 3.52186832248982, 3.48385687456328, 3.50846686334877, 3.58371795445638, - 3.80788075066933, 3.793359430218, 3.80935514961242, 3.83281418047548, - 3.84205468437419, 3.83178045932269, 3.75630308120772, 3.73671407998881, - 3.6873827382772, 3.66288489121358, 3.68738273866529, 3.73671408030886, - 3.83178045917173, 3.84205468411383, 3.83281418003925, 3.8093551489737, - 3.79335942975339, 3.80788075045042, 2.27781169371947, 2.35355625792149, - 2.29919933349799, 2.16556407711254, 2.07741510386896, 2.14206673667462, - 2.47890799981366, 2.5433708395083, 2.67586026364294, 2.7303060630097, - 2.67586026267281, 2.54337083843576, 2.35355626028493, 2.27781169484832, - 2.14206673797471, 2.07741510630394, 2.16556408100954, 2.29919933687183, - 3.73671407846663, 3.75630307940097, 3.73671407836399, 3.68738273582522, - 3.66288488872016, 3.6873827359458, 3.79335942881371, 3.80788074939396, - 3.83178045860985, 3.84205468362, 3.83281417958652, 3.80935514809748, - 3.83178045866981, 3.80788074946883, 3.793359428979, 3.80935514832405, - 3.8328141797671, 3.84205468371904, 3.80788074832992, 3.83178045753363, - 3.84205468286234, 3.83281417846865, 3.80935514701321, 3.79335942734603, - 3.83178045763168, 3.80788074840196, 3.79335942757774, 3.80935514723929, - 3.83281417876637, 3.84205468298099, 3.73671407681723, 3.75630307816502, - 3.73671407670314, 3.68738273458223, 3.66288488699152, 3.68738273464747, - 2.27781168468752, 2.35355624971977, 2.29919932524649, 2.16556406900792, - 2.07741509487063, 2.14206672772056, 2.54337083020097, 2.47890799077767, - 2.54337083048016, 2.67586025464235, 2.73030605480022, 2.67586025443279, - 2.35355625050973, 2.2778116851111, 2.14206672824714, 2.07741509602769, - 2.16556407062235, 2.29919932655198, 3.75630308447383, 3.73671408351201, - 3.68738274269091, 3.66288489561696, 3.68738274275124, 3.73671408355243, - 3.79335943233127, 3.80788075280813, 3.83178046073998, 3.84205468551873, - 3.8328141816563, 3.8093551514171, 3.83178046075959, 3.80788075283164, - 3.79335943239049, 3.80935515149345, 3.83281418171944, 3.84205468554513, - 3.80788074787498, 3.83178045717735, 3.84205468265246, 3.83281417831282, - 3.80935514676775, 3.79335942695681, 3.83178045708815, 3.80788074775654, - 3.79335942669308, 3.80935514642365, 3.83281417805311, 3.84205468250755, - 3.73671407601969, 3.68738273376602, 3.66288488597554, 3.68738273356469, - 3.73671407584276, 3.75630307741336, 2.27781168194595, 2.35355624725223, - 2.29919932326004, 2.16556406678664, 2.07741509233704, 2.14206672464747, - 2.54337082758285, 2.47890798793938, 2.54337082725197, 2.6758602517937, - 2.7303060521297, 2.67586025212544, 2.35355624667329, 2.27781168170246, - 2.14206672440549, 2.07741509198883, 2.16556406611879, 2.29919932257358, - 2.27781168284262, 2.14206672539865, 2.07741509274297, 2.16556406687647, - 2.29919932348106, 2.35355624761155, 2.14206672623215, 2.27781168362573, - 2.3535562489834, 2.29919932583701, 2.16556406939985, 2.07741509459602, - 2.54337082892105, 2.47890798945701, 2.54337082836692, 2.67586025344429, - 2.73030605344056, 2.67586025401124, 2.54337084690776, 2.47890800846272, - 2.5433708462933, 2.67586027161076, 2.73030607102055, 2.67586027222717, - 2.35355626523264, 2.27781170199606, 2.14206674320176, 2.07741511130966, - 2.16556408348771, 2.29919934142396, 2.35355626673157, 2.29919934411479, - 2.16556408615858, 2.07741511345627, 2.14206674410725, 2.27781170290775, - 3.75630307940099, 3.73671407846663, 3.68738273594581, 3.66288488872017, - 3.68738273582524, 3.73671407836399, 3.80788074946884, 3.83178045866981, - 3.84205468371904, 3.8328141797671, 3.80935514832406, 3.793359428979, - 3.79335942881371, 3.80935514809749, 3.83281417958652, 3.84205468362001, - 3.83178045860986, 3.80788074939397, 3.75630307816503, 3.73671407681723, - 3.68738273464747, 3.66288488699152, 3.68738273458223, 3.73671407670314, - 3.80788074840196, 3.83178045763168, 3.84205468298099, 3.83281417876637, - 3.80935514723929, 3.79335942757774, 3.83178045753363, 3.80788074832992, - 3.79335942734603, 3.80935514701322, 3.83281417846865, 3.84205468286235, - 3.793359430218, 3.80788075066933, 3.83178045932269, 3.84205468437419, - 3.83281418047548, 3.80935514961242, 3.83178045917173, 3.80788075045042, - 3.79335942975339, 3.8093551489737, 3.83281418003925, 3.84205468411383, - 3.7367140799888, 3.75630308120773, 3.73671408030886, 3.68738273866529, - 3.66288489121358, 3.6873827382772, 2.27781169484829, 2.35355626028491, - 2.29919933687181, 2.16556408100954, 2.07741510630392, 2.14206673797472, - 2.54337083950829, 2.47890799981363, 2.54337083843575, 2.6758602626728, - 2.7303060630097, 2.67586026364293, 2.35355625792149, 2.27781169371944, - 2.14206673667463, 2.07741510386894, 2.16556407711255, 2.29919933349798, - 2.35355624971976, 2.27781168468751, 2.14206672772055, 2.07741509487061, - 2.16556406900791, 2.29919932524647, 2.27781168511109, 2.35355625050974, - 2.29919932655197, 2.16556407062236, 2.07741509602769, 2.14206672824714, - 2.54337083020097, 2.67586025443278, 2.73030605480022, 2.67586025464233, - 2.54337083048016, 2.47890799077766, 3.80788075283164, 3.83178046075959, - 3.84205468554513, 3.83281418171944, 3.80935515149345, 3.79335943239049, - 3.80788075280813, 3.79335943233126, 3.8093551514171, 3.8328141816563, - 3.84205468551873, 3.83178046073998, 3.73671408351201, 3.75630308447383, - 3.73671408355242, 3.68738274275125, 3.66288489561696, 3.6873827426909, - 2.35355624725224, 2.27781168194597, 2.14206672464747, 2.07741509233704, - 2.16556406678664, 2.29919932326005, 2.27781168170248, 2.3535562466733, - 2.2991993225736, 2.16556406611881, 2.07741509198885, 2.1420667244055, - 2.54337082758287, 2.67586025212545, 2.73030605212971, 2.67586025179371, - 2.54337082725199, 2.47890798793941, 3.83178045717735, 3.80788074787498, - 3.79335942695681, 3.80935514676775, 3.83281417831282, 3.84205468265246, - 3.73671407601969, 3.75630307741336, 3.73671407584277, 3.68738273356468, - 3.66288488597554, 3.68738273376602, 3.83178045708815, 3.84205468250755, - 3.83281417805311, 3.80935514642365, 3.79335942669308, 3.80788074775654, - 2.27781170199606, 2.35355626523265, 2.29919934142398, 2.16556408348772, - 2.07741511130968, 2.14206674320177, 2.47890800846272, 2.54337084690776, - 2.67586027222717, 2.73030607102056, 2.67586027161076, 2.5433708462933, - 2.35355626673158, 2.27781170290776, 2.14206674410724, 2.07741511345628, - 2.16556408615858, 2.2991993441148, 2.27781168362574, 2.14206672623214, - 2.07741509459602, 2.16556406939983, 2.29919932583703, 2.3535562489834, - 2.14206672539864, 2.27781168284262, 2.35355624761154, 2.29919932348108, - 2.16556406687647, 2.07741509274298, 2.54337082892103, 2.67586025401125, - 2.73030605344055, 2.67586025344429, 2.54337082836691, 2.47890798945702, - 3.56582381262452, 3.49179372077245, 3.35098504543044, 3.35098504600089, - 3.49179372147866, 3.49179371820619, 3.5658238094764, 3.49179371789595, - 3.35098504184857, 3.35098504242207, 0.27021038919352, 0.270210381080679, - 0.270210376352625, 0.270210379799991, 0.270210389398549, - 3.49179372147868, 3.35098504600089, 3.35098504543044, 3.49179372077246, - 3.56582381262451, 3.49179371789595, 3.5658238094764, 3.49179371820619, - 3.35098504242206, 3.35098504184857, 3.49179372913151, 3.35098505489057, - 3.35098505425721, 3.49179372833768, 3.56582381947488, 3.35098504458005, - 3.49179372001333, 3.56582381207765, 3.49179372042905, 3.35098504457686, - 3.49179372042905, 3.56582381207766, 3.49179372001333, 3.35098504458005, - 3.35098504457686, 0.27021038939853, 0.270210379800009, 0.270210376352618, - 0.270210381080673, 0.270210389193507, 3.49179372435823, 3.56582381677044, - 3.49179372630058, 3.3509850510875, 3.35098504989944, 3.49179372630059, - 3.56582381677044, 3.49179372435824, 3.35098504989946, 3.3509850510875, - 3.35098505425721, 3.35098505489056, 3.49179372913152, 3.56582381947489, - 3.49179372833769, 3.19535085205332, 3.07668323326383, 3.19535085352684, - 3.37359787129475, 3.46244566403506, 3.37359787006572, 3.84736470560324, - 3.85625299887573, 3.85830508739538, 3.84736470580818, 3.85625299898753, - 3.85830508731231, 3.85830508724626, 3.85625299905455, 3.84736470599368, - 3.8583050875722, 3.85625299862475, 3.84736470530559, 3.37359786470753, - 3.19535084579305, 3.46244565827196, 3.37359786459013, 3.19535084565499, - 3.07668322603769, 2.26471331729362, 2.0537377418623, 1.79512453701271, - 1.7951245390242, 2.05373774457892, 2.264713318928, 3.84736470479973, - 3.85625299829885, 3.85830508781515, 3.84736470699574, 3.8562529996109, - 3.85830508681783, 3.85830508677252, 3.85625299968159, 3.84736470714796, - 3.85830508792246, 3.8562529981672, 3.84736470458645, 3.37359786276103, - 3.46244565686712, 3.37359786333627, 3.1953508442598, 3.07668322389714, - 3.19535084354769, 2.26471331183879, 2.0537377360699, 1.79512453087718, - 1.7951245314102, 2.05373773758979, 2.26471331239681, 2.26471332332703, - 2.05373774888787, 1.79512454228193, 1.79512453798091, 2.05373774273307, - 2.26471331979257, 1.79512454988451, 1.79512454734629, 2.05373775476819, - 2.2647133307081, 2.26471332835024, 2.05373775305884, 1.7951245306023, - 2.0537377365977, 2.26471331244287, 2.26471331230658, 2.05373773648767, - 1.79512453038448, 3.84736470530559, 3.85830508724626, 3.85625299862475, - 3.8583050875722, 3.84736470599368, 3.85625299905455, 3.85830508739538, - 3.84736470580818, 3.85625299887573, 3.84736470560324, 3.85830508731231, - 3.85625299898753, 3.19535085352682, 3.07668323326383, 3.1953508520533, - 3.37359787006572, 3.46244566403506, 3.37359787129476, 2.264713318928, - 2.05373774457893, 1.79512453902421, 1.79512453701272, 2.05373774186231, - 2.26471331729362, 3.19535084579303, 3.07668322603768, 3.19535084565498, - 3.37359786459013, 3.46244565827196, 3.37359786470753, 3.85830508681783, - 3.8562529996109, 3.84736470699574, 3.85830508781515, 3.85625299829884, - 3.84736470479973, 1.79512453087719, 2.0537377360699, 2.2647133118388, - 2.2647133123968, 2.05373773758979, 1.79512453141019, 3.37359786333627, - 3.46244565686712, 3.37359786276104, 3.1953508435477, 3.07668322389715, - 3.1953508442598, 3.84736470458645, 3.85830508677252, 3.8562529981672, - 3.85830508792246, 3.84736470714795, 3.85625299968159, 3.19535086027984, - 3.07668324119428, 3.19535086082623, 3.37359787765526, 3.46244567038539, - 3.37359787730248, 2.2647133307081, 2.05373775476817, 1.79512454988451, - 1.79512454734627, 2.05373775305883, 2.26471332835022, 3.84736470659195, - 3.85830508775007, 3.85625299931045, 3.8583050870498, 3.84736470503321, - 3.85625299838543, 3.1953508468231, 3.07668322562571, 3.19535084617094, - 3.37359786505137, 3.46244565973068, 3.37359786545367, 1.79512453038448, - 2.05373773648765, 2.26471331230659, 2.26471331244288, 2.0537377365977, - 1.7951245306023, 3.19535084617094, 3.0766832256257, 3.1953508468231, - 3.37359786545366, 3.46244565973068, 3.37359786505135, 3.85625299838543, - 3.84736470503321, 3.8583050870498, 3.85625299931045, 3.84736470659195, - 3.85830508775007, 1.79512454228193, 2.05373774888787, 2.26471332332703, - 2.26471331979257, 2.05373774273307, 1.79512453798092, 3.37359787730248, - 3.19535086027983, 3.46244567038539, 3.37359787765525, 3.19535086082623, - 3.07668324119428, 3.49962488067984, 3.43713675798983, 3.2772942422533, - 3.14070016124613, 3.22380484984836, 3.39577856947789, 3.65861887353402, - 3.69666641030478, 3.77954577861335, 3.81231167451188, 3.79051717653684, - 3.71291500127558, 3.71291500077803, 3.79051717621703, 3.81231167402121, - 3.77954577760484, 3.69666640880429, 3.65861887241131, 3.14070015788068, - 3.27729423978309, 3.43713675573172, 3.49962487776198, 3.39577856561065, - 3.22380484540448, 3.05128051322848, 2.88592335054783, 2.68698336414128, - 2.68698336541731, 2.88592335215934, 3.05128051414338, 3.0512805090478, - 2.88592334617744, 2.68698335936481, 2.6869833600004, 2.88592334721774, - 3.05128050974972, 3.49962487671691, 3.43713675423585, 3.27729423807334, - 3.14070015643117, 3.22380484474151, 3.3957785650321, 3.79051717462295, - 3.71291499836736, 3.65861887072152, 3.69666640720477, 3.77954577649199, - 3.8123116724299, 3.69666640659036, 3.65861887018596, 3.71291499823603, - 3.79051717450862, 3.81231167229886, 3.77954577603718, 3.43713675297417, - 3.49962487520551, 3.39577856316439, 3.22380484250529, 3.14070015453916, - 3.27729423642931, 0.77165547598209, 1.07623094619881, 1.29180705383508, - 1.29180705371894, 1.07623094762527, 0.771655476539705, 0.771655482204467, - 1.07623095174485, 1.29180706091197, 1.29180706584451, 1.07623095935429, - 0.771655488096443, 0.771655493061666, 1.07623096303874, 1.29180707275934, - 1.29180707050859, 1.07623096220338, 0.771655491577014, 0.77165548712907, - 1.07623095806281, 1.29180706380607, 1.2918070612426, 1.07623095277605, - 0.771655483333819, 0.771655478205974, 0.771655476779375, - 1.07623094945772, 1.29180705531781, 1.29180705463828, 1.07623094668678, - 3.27729423978309, 3.14070015788068, 3.22380484540448, 3.39577856561065, - 3.49962487776198, 3.43713675573173, 3.65861887241132, 3.69666640880429, - 3.77954577760484, 3.81231167402121, 3.79051717621703, 3.71291500077803, - 3.69666641030478, 3.65861887353403, 3.71291500127557, 3.79051717653685, - 3.81231167451188, 3.77954577861336, 3.43713675798985, 3.49962488067984, - 3.3957785694779, 3.22380484984836, 3.14070016124615, 3.2772942422533, - 3.05128051414338, 2.88592335215934, 2.68698336541731, 2.68698336414127, - 2.88592335054783, 3.05128051322848, 3.71291499836736, 3.79051717462295, - 3.8123116724299, 3.77954577649199, 3.69666640720476, 3.65861887072152, - 3.43713675423583, 3.49962487671691, 3.3957785650321, 3.22380484474149, - 3.14070015643116, 3.27729423807334, 2.88592334617744, 3.05128050904779, - 3.05128050974972, 2.88592334721774, 2.68698336000039, 2.68698335936481, - 3.49962487520551, 3.43713675297416, 3.27729423642931, 3.14070015453917, - 3.22380484250529, 3.3957785631644, 3.65861887018596, 3.69666640659035, - 3.77954577603718, 3.81231167229886, 3.79051717450862, 3.71291499823603, - 3.43713676607313, 3.49962488811077, 3.39577857766595, 3.22380485941718, - 3.14070017107195, 3.27729425170688, 3.05128052482442, 2.88592336299732, - 2.68698337712499, 2.68698337528081, 2.88592336141154, 3.05128052362768, - 3.27729424882958, 3.14070016717738, 3.22380485410074, 3.39577857302032, - 3.49962488482424, 3.43713676345293, 3.65861887797203, 3.69666641359204, - 3.77954578099049, 3.81231167644173, 3.79051717945454, 3.71291500542055, - 3.71291500595043, 3.79051717980798, 3.81231167698461, 3.77954578211128, - 3.69666641520302, 3.65861887922611, 3.79051717588132, 3.71291500028723, - 3.65861887194212, 3.69666640821601, 3.77954577718953, 3.81231167350748, - 3.69666640957692, 3.65861887271224, 3.71291500086573, 3.79051717623123, - 3.81231167422142, 3.77954577809428, 3.43713675581096, 3.49962487854547, - 3.39577856581431, 3.22380484540421, 3.14070015711114, 3.27729423946488, - 2.88592334824201, 3.05128051151309, 3.0512805114258, 2.88592334836884, - 2.68698336090578, 2.68698336098713, 3.27729423901026, 3.14070015657375, - 3.22380484442033, 3.39577856469585, 3.49962487716159, 3.43713675500253, - 3.14070015657376, 3.27729423901026, 3.43713675500254, 3.49962487716159, - 3.39577856469585, 3.22380484442034, 3.05128051151309, 2.88592334824202, - 2.68698336098714, 2.6869833609058, 2.88592334836885, 3.05128051142581, - 3.49962487854547, 3.43713675581094, 3.27729423946488, 3.14070015711113, - 3.22380484540421, 3.39577856581429, 3.65861887271223, 3.69666640957692, - 3.77954577809428, 3.81231167422143, 3.79051717623123, 3.71291500086573, - 3.71291500028723, 3.79051717588132, 3.81231167350748, 3.77954577718954, - 3.69666640821602, 3.65861887194212, 0.771655476779391, 1.07623094668678, - 1.2918070546383, 1.2918070553178, 1.07623094945772, 0.771655478205971, - 0.771655483333827, 1.07623095277605, 1.2918070612426, 1.29180706380607, - 1.07623095806282, 0.77165548712907, 0.771655491576999, 1.07623096220337, - 1.29180707050858, 1.29180707275935, 1.07623096303872, 0.771655493061662, - 0.771655488096436, 1.07623095935428, 1.29180706584451, 1.29180706091197, - 1.07623095174483, 0.771655482204467, 0.771655476539703, 1.07623094762526, - 1.29180705371894, 1.2918070538351, 1.07623094619881, 0.771655475982092, - 3.43713676411029, 3.49962488695124, 3.39577857655324, 3.22380485775483, - 3.14070016865676, 3.27729424892005, 2.88592335363144, 3.05128051782405, - 3.05128051957792, 2.88592335841908, 2.68698337060877, 2.68698336799196, - 3.49962488024322, 3.43713675823407, 3.27729424299861, 3.14070015965667, - 3.22380484741502, 3.39577856710617, 3.65861887496794, 3.69666641108682, - 3.77954577923313, 3.8123116754869, 3.7905171781339, 3.71291500353373, - 3.71291500461778, 3.65861887776378, 3.79051717887979, 3.81231167643212, - 3.77954578151993, 3.6966664143527, 3.65861887776378, 3.6966664143527, - 3.77954578151993, 3.81231167643212, 3.79051717887978, 3.71291500461778, - 3.71291500353374, 3.7905171781339, 3.81231167548691, 3.77954577923312, - 3.69666641108682, 3.65861887496794, 3.43713675823408, 3.49962488024323, - 3.39577856710617, 3.22380484741503, 3.14070015965667, 3.27729424299862, - 3.05128051782406, 2.88592335363145, 2.68698336799197, 2.68698337060877, - 2.88592335841909, 3.05128051957793, 3.27729424892005, 3.43713676411029, - 3.14070016865677, 3.22380485775483, 3.39577857655324, 3.49962488695124, - 3.79051717980798, 3.71291500595043, 3.65861887922611, 3.69666641520302, - 3.77954578211128, 3.81231167698461, 3.69666641359205, 3.65861887797203, - 3.71291500542055, 3.79051717945454, 3.81231167644173, 3.77954578099048, - 3.14070016717738, 3.2772942488296, 3.43713676345293, 3.49962488482424, - 3.39577857302032, 3.22380485410075, 2.88592336299733, 3.05128052482443, - 3.0512805236277, 2.88592336141155, 2.68698337528083, 2.686983377125, - 3.2772942517069, 3.43713676607314, 3.14070017107196, 3.22380485941718, - 3.39577857766596, 3.49962488811077, 3.60710580370843, 3.67635828474096, - 3.76574272955737, 3.79577624852149, 3.7574028163651, 3.6644417983105, - 3.79577624763033, 3.7574028157606, 3.76574272789331, 3.6763582824941, - 3.60710580188714, 3.66444179750507, 2.88150376323689, 2.78666042350304, - 2.55573223277098, 2.38227207627246, 2.48760942421827, 2.73664139789877, - 2.38227208099292, 2.48760942630927, 2.55573223893605, 2.78666042924593, - 2.88150376677042, 2.73664139983553, 3.76574272689076, 3.67635828108227, - 3.60710579922893, 3.66444179533085, 3.75740281419147, 3.79577624688206, - 3.85454389952957, 3.84162713047746, 3.85934573156789, 3.85247873751497, - 3.859345731605, 3.85454389946519, 3.85454389979809, 3.84162713106653, - 3.85934573142924, 3.85247873734865, 3.85934573150589, 3.8545438996606, - 3.85934573183683, 3.85247873823256, 3.859345731896, 3.85454389888442, - 3.84162712973063, 3.85454389900258, 3.66444179380486, 3.75740281310853, - 3.60710579835928, 3.67635827990139, 3.76574272607734, 3.79577624582888, - 3.66444179558235, 3.75740281438795, 3.60710579982856, 3.67635828188469, - 3.76574272751114, 3.79577624721842, 2.88150375660528, 2.78666041752384, - 2.55573222606169, 2.38227206890193, 2.48760941595645, 2.73664139047317, - 2.3822720705657, 2.48760941657267, 2.55573222819302, 2.78666041906537, - 2.88150375726034, 2.73664139087425, 3.79577624534713, 3.75740281296004, - 3.76574272552478, 3.67635827912559, 3.60710579807384, 3.66444179359528, - 1.94256929533975, 2.06419703422539, 1.64625341329628, 1.49258032362887, - 1.64625341618252, 1.94256929819578, 1.94256928652913, 2.06419702451545, - 1.64625340375581, 1.49258031377706, 1.64625340573984, 1.94256928791764, - 3.7657427313086, 3.67635828714392, 3.60710580747812, 3.66444180217202, - 3.75740281922419, 3.79577625024055, 3.85454390039865, 3.84162713238925, - 3.85934573111434, 3.85247873669336, 3.8593457311224, 3.85454390037747, - 3.66444180228992, 3.75740281930348, 3.60710580786064, 3.67635828756557, - 3.76574273159901, 3.79577625037159, 3.85934573200796, 3.85247873855185, - 3.85934573196111, 3.85454389875805, 3.84162712938197, 3.8545438986714, - 3.6644417924823, 3.75740281213609, 3.60710579668096, 3.67635827805925, - 3.76574272474145, 3.79577624472912, 3.79577624523555, 3.75740281245502, - 3.76574272559159, 3.67635827923429, 3.60710579757978, 3.66444179291844, - 2.88150375540616, 2.78666041686582, 2.55573222538432, 2.38227206724844, - 2.48760941375912, 2.73664138838674, 2.38227206619357, 2.48760941318099, - 2.55573222394602, 2.78666041513284, 2.88150375406378, 2.7366413877045, - 1.94256928368544, 2.06419702117996, 1.64625340097009, 1.49258030946353, - 1.64625340095025, 1.94256928336294, 1.64625340403913, 1.49258031157361, - 1.64625340140512, 1.94256928417001, 2.06419702248973, 1.9425692861929, - 2.73664138992618, 2.48760941579546, 2.88150375806085, 2.78666041976306, - 2.5557322293941, 2.3822720699552, 2.55573224014291, 2.78666043118354, - 2.88150377196666, 2.7366414061738, 2.48760943289103, 2.38227208328737, - 1.94256930302161, 2.06419704184882, 1.64625342168003, 1.49258032708336, - 1.64625341877086, 1.94256930089721, 2.73664140715473, 2.48760943439797, - 2.88150377390757, 2.78666043406952, 2.55573224445283, 2.38227208623046, - 2.3822720671636, 2.48760941446217, 2.55573222531352, 2.7866604169237, - 2.8815037561134, 2.73664138903275, 3.76574272751114, 3.79577624721842, - 3.67635828188469, 3.60710579982857, 3.66444179558235, 3.75740281438796, - 3.85454389946519, 3.84162713047747, 3.859345731605, 3.85247873751497, - 3.85934573156789, 3.85454389952957, 3.76574272607734, 3.67635827990139, - 3.60710579835928, 3.66444179380487, 3.75740281310853, 3.79577624582888, - 3.85454389888442, 3.84162712973063, 3.859345731896, 3.85247873823256, - 3.85934573183683, 3.85454389900258, 3.85934573150589, 3.85247873734865, - 3.85934573142924, 3.85454389979809, 3.84162713106653, 3.8545438996606, - 3.66444179750507, 3.7574028157606, 3.60710580188714, 3.6763582824941, - 3.76574272789332, 3.79577624763033, 3.66444179533086, 3.75740281419148, - 3.60710579922895, 3.67635828108227, 3.76574272689076, 3.79577624688206, - 2.73664139983553, 2.48760942630926, 2.8815037667704, 2.78666042924592, - 2.55573223893604, 2.38227208099292, 2.38227207627247, 2.48760942421826, - 2.55573223277098, 2.78666042350305, 2.88150376323689, 2.73664139789877, - 3.79577624852149, 3.7574028163651, 3.76574272955737, 3.67635828474096, - 3.60710580370842, 3.6644417983105, 1.64625340573983, 1.49258031377706, - 1.64625340375579, 1.94256928652912, 2.06419702451544, 1.94256928791765, - 2.73664139087425, 2.48760941657266, 2.88150375726033, 2.78666041906536, - 2.55573222819301, 2.38227207056571, 1.94256929819578, 2.06419703422538, - 1.64625341618251, 1.49258032362887, 1.64625341329627, 1.94256929533975, - 3.66444179359528, 3.75740281296004, 3.60710579807383, 3.67635827912559, - 3.76574272552478, 3.79577624534713, 2.38227206890191, 2.48760941595643, - 2.55573222606167, 2.78666041752383, 2.88150375660526, 2.73664139047317, - 3.8593457311224, 3.85247873669336, 3.85934573111434, 3.85454390039864, - 3.84162713238925, 3.85454390037747, 3.66444180217202, 3.75740281922419, - 3.60710580747812, 3.67635828714392, 3.7657427313086, 3.79577625024055, - 1.64625340095026, 1.94256928336295, 1.49258030946354, 1.64625340097011, - 1.94256928368545, 2.06419702117997, 2.7366413877045, 2.48760941318101, - 2.88150375406379, 2.78666041513284, 2.55573222394604, 2.38227206619358, - 2.38227206724845, 2.48760941375913, 2.55573222538432, 2.78666041686584, - 2.88150375540617, 2.73664138838674, 3.67635827923429, 3.76574272559159, - 3.79577624523555, 3.75740281245502, 3.66444179291844, 3.60710579757979, - 3.79577624472912, 3.75740281213609, 3.76574272474145, 3.67635827805925, - 3.60710579668096, 3.6644417924823, 3.85454389875805, 3.84162712938197, - 3.85934573196111, 3.85247873855186, 3.85934573200796, 3.8545438986714, - 3.79577625037158, 3.75740281930348, 3.76574273159901, 3.67635828756557, - 3.60710580786063, 3.66444180228992, 2.7366414061738, 2.48760943289103, - 2.88150377196667, 2.78666043118355, 2.55573224014291, 2.38227208328738, - 2.38227208623045, 2.48760943439797, 2.55573224445283, 2.78666043406951, - 2.88150377390757, 2.73664140715472, 1.9425693008972, 2.06419704184881, - 1.64625341877087, 1.49258032708335, 1.64625342168003, 1.94256930302161, - 1.64625340140513, 1.49258031157358, 1.64625340403914, 1.94256928619291, - 2.06419702248974, 1.94256928417, 2.73664138903273, 2.48760941446219, - 2.8815037561134, 2.7866604169237, 2.55573222531353, 2.38227206716359, - 2.3822720699552, 2.48760941579547, 2.5557322293941, 2.78666041976306, - 2.88150375806086, 2.73664138992617, 3.53977331774956, 3.21649654835339, - 3.21649654959376, 3.53977331919744, 3.67793587513922, 3.21649654540194, - 3.53977331564885, 3.67793587277307, 3.5397733149207, 3.21649654477022, - 0.593850783950195, 0.593850779396398, 0.593850782775934, - 0.593850791659571, 0.593850791598252, 3.67793587513923, 3.53977331919744, - 3.21649654959376, 3.21649654835339, 3.53977331774956, 3.53977331564884, - 3.21649654540194, 3.21649654477022, 3.5397733149207, 3.67793587277307, - 3.21649655924389, 3.21649655790655, 3.53977332472888, 3.67793588059418, - 3.53977332629927, 3.21649654702879, 3.53977331720081, 3.6779358746755, - 3.53977331812552, 3.21649654752238, 3.53977331812553, 3.6779358746755, - 3.53977331720082, 3.21649654702879, 3.21649654752239, 0.59385078277592, - 0.593850779396409, 0.593850783950208, 0.593850791598261, - 0.593850791659587, 3.21649655188639, 3.53977332093464, 3.6779358784194, - 3.53977332442296, 3.21649655569418, 3.6779358784194, 3.53977332093465, - 3.21649655188639, 3.21649655569419, 3.53977332442296, 3.53977332472888, - 3.21649655790655, 3.2164965592439, 3.53977332629928, 3.67793588059419, - 3.09313226062383, 2.85913196185193, 3.09313226370159, 3.47211595370855, - 3.59954851044558, 3.47211595146765, 3.80410697369584, 3.84328455748379, - 3.85117147164709, 3.80410697328136, 3.84328455709745, 3.85117147178522, - 3.85117147158168, 3.84328455729499, 3.80410697400469, 3.85117147224626, - 3.84328455656832, 3.80410697261409, 3.47211594706444, 3.59954850580681, - 3.4721159467236, 3.09313225474967, 2.85913195368211, 3.09313225521847, - 2.49541414306506, 2.11642434923235, 1.57437664082969, 1.57437664453741, - 2.11642435506745, 2.4954141451353, 3.80410697156069, 3.84328455632404, - 3.85117147267505, 3.80410697604156, 3.84328455888855, 3.85117147088586, - 3.85117147081093, 3.84328455894811, 3.8041069763742, 3.85117147291742, - 3.84328455598842, 3.80410697116346, 3.47211594499978, 3.59954850464263, - 3.4721159461514, 3.09313225360166, 2.85913195134544, 3.09313225235955, - 2.49541413793282, 2.11642434460638, 1.57437663462051, 1.57437663650702, - 2.11642434687757, 2.49541413924946, 2.11642436178007, 1.57437664731759, - 1.57437664054781, 2.11642434882777, 2.49541414518076, 2.49541415071087, - 1.57437665272758, 2.11642436593707, 2.49541415619563, 2.49541415444931, - 2.11642435990732, 1.57437665132578, 1.57437663439456, 1.57437663476153, - 2.11642434521013, 2.49541413934613, 2.4954141391958, 2.11642434512195, - 3.84328455729499, 3.85117147158168, 3.80410697261409, 3.84328455656833, - 3.85117147224626, 3.80410697400469, 3.85117147178522, 3.84328455709745, - 3.80410697328136, 3.85117147164709, 3.84328455748379, 3.80410697369584, - 2.85913196185193, 3.09313226062383, 3.47211595146765, 3.59954851044558, - 3.47211595370855, 3.09313226370158, 2.49541414513532, 2.11642435506745, - 1.57437664453741, 1.5743766408297, 2.11642434923235, 2.49541414306506, - 3.09313225521847, 2.8591319536821, 3.09313225474966, 3.47211594672359, - 3.59954850580681, 3.47211594706444, 3.84328455632404, 3.80410697156069, - 3.85117147088586, 3.84328455888855, 3.80410697604156, 3.85117147267505, - 2.1164243446064, 2.49541413793282, 2.49541413924946, 2.11642434687755, - 1.57437663650702, 1.5743766346205, 3.09313225235954, 2.85913195134545, - 3.09313225360167, 3.4721159461514, 3.59954850464264, 3.47211594499978, - 3.80410697116346, 3.84328455598842, 3.85117147291742, 3.8041069763742, - 3.84328455894811, 3.85117147081093, 2.85913197060393, 3.09313227004965, - 3.47211595876244, 3.59954851569115, 3.47211595823572, 3.09313226885001, - 2.49541415619561, 2.11642436593707, 1.57437665272757, 1.57437665132577, - 2.1164243599073, 2.4954141544493, 3.80410697522113, 3.84328455797478, - 3.85117147135552, 3.80410697204667, 3.84328455626233, 3.85117147268263, - 2.85913195335129, 3.09313225443108, 3.47211594813884, 3.59954850695455, - 3.47211594893677, 3.09313225559275, 2.11642434512197, 2.49541413919581, - 2.49541413934613, 2.11642434521014, 1.57437663439455, 1.57437663476152, - 3.09313225443106, 2.85913195335131, 3.09313225559274, 3.47211594893677, - 3.59954850695453, 3.47211594813884, 3.85117147135552, 3.80410697204667, - 3.84328455797478, 3.80410697522113, 3.85117147268263, 3.84328455626233, - 1.57437664731759, 2.11642436178009, 2.49541415071088, 2.49541414518077, - 2.11642434882777, 1.57437664054782, 3.47211595823571, 3.59954851569115, - 3.47211595876244, 3.09313227004964, 2.85913197060393, 3.09313226885001, - 2.9465344275923, 3.62843002484419, 3.81515503045961, 3.62843002252828, - 2.94653442409549, 2.94653442027827, 2.94653442228269, 3.62843002165761, - 3.81515502849687, 3.62843002048158, 1.17516569271611, 1.1751656948359, - 1.17516571106594, 1.17516570731716, 1.17516569707796, 2.94653442409548, - 3.62843002252828, 3.8151550304596, 3.62843002484421, 2.94653442759231, - 3.81515502849687, 3.62843002165761, 2.94653442228267, 2.94653442027828, - 3.62843002048158, 3.62843003086443, 2.94653443870383, 2.94653443438414, - 3.62843002819084, 3.81515503282455, 3.62843002199762, 3.81515503007801, - 3.62843002318803, 2.94653442262583, 2.94653442250223, 3.62843002199762, - 2.94653442250223, 2.94653442262583, 3.62843002318802, 3.81515503007802, - 1.17516569271612, 1.17516569707796, 1.17516570731713, 1.17516571106595, - 1.17516569483589, 3.62843002999132, 2.94653443494701, 2.94653442666621, - 3.62843002450252, 3.81515503209295, 3.62843002999132, 3.81515503209295, - 3.62843002450252, 2.94653442666622, 2.94653443494701, 3.62843003086443, - 3.81515503282455, 3.62843002819084, 2.94653443438416, 2.94653443870384, - 3.79252788302589, 2.34391114276879, 3.79252788191438, 3.79252788048945, - 2.34391113374791, 3.79252788545175, 3.79252787984217, 2.3439111305481, - 2.3439111317078, 2.34391114991211, 3.79252788191438, 3.79252788048945, - 3.79252788302589, 2.34391114276878, 2.34391113374791, 3.79252788545175, - 2.34391113054811, 3.79252787984217, 2.34391114991211, 2.34391113170778 ; - - vVelocity = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ; - - iceAreaCell = 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, - 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, - 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0 ; - - iceVolumeCell = 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, - 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, - 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0 ; - - iceAreaCategory = - 0, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 0, - 0, - 0, - 0, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 0, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 0, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 0, - 0, - 0, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 0, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 0, - 0, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 0, - 0, - 0, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 0, - 1, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 0, - 0, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 0, - 0, - 0, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 0, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 0, - 0, - 0, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0 ; - - iceVolumeCategory = - 0, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 0, - 0, - 0, - 0, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 0, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 0, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 0, - 0, - 0, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 0, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 0, - 0, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 0, - 0, - 0, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 0, - 1, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 0, - 0, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 0, - 0, - 0, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 0, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 1, - 1, - 0, - 0, - 0, - 1, - 1, - 1, - 0, - 0, - 0, - 0, - 0, - 0 ; -} diff --git a/test/testFortranMPReconstruction.f90 b/test/testFortranMPReconstruction.f90 index 1253265..03a4165 100644 --- a/test/testFortranMPReconstruction.f90 +++ b/test/testFortranMPReconstruction.f90 @@ -186,6 +186,8 @@ program main deallocate(meshVtxMass) deallocate(meshVtxMass1) deallocate(meshElmMass) + deallocate(meshVtxVelu) + deallocate(meshVtxVelv) stop From 5da1001209d444ab86dfd86b2c3981dcf818d754 Mon Sep 17 00:00:00 2001 From: dhyan1272 Date: Fri, 14 Feb 2025 14:12:41 -0800 Subject: [PATCH 05/42] Order 1 reconstruction, coefficients calculated once --- src/pmpo_MPMesh.cpp | 1 + src/pmpo_MPMesh.hpp | 12 +++++- src/pmpo_MPMesh_assembly.hpp | 83 ++++++++++++++++++++++++------------ 3 files changed, 68 insertions(+), 28 deletions(-) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index 45319be..a5f709a 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -301,6 +301,7 @@ void MPMesh::reconstructSlices() { if (reconstructSlice.size() == 0) return; Kokkos::Timer timer; calcBasis(); + resetPreComputeFlag(); for (auto const& [index, reconstruct] : reconstructSlice) { if (reconstruct) reconstruct(); } diff --git a/src/pmpo_MPMesh.hpp b/src/pmpo_MPMesh.hpp index 5a63ef9..33f0599 100644 --- a/src/pmpo_MPMesh.hpp +++ b/src/pmpo_MPMesh.hpp @@ -16,7 +16,17 @@ template <> const MaterialPointSlice meshFieldIndexToMPSlice < MeshF_RotLatLonIn #define maxMPsPerElm 8 class MPMesh{ + private: + + bool isPreComputed; + public: + + MPMesh() : isPreComputed(false){}; + void computeMatricesAndSolve(); + void resetPreComputeFlag(); + Kokkos::View precomputedVtxCoeffs; + Mesh* p_mesh; MaterialPoints* p_MPs; @@ -49,7 +59,7 @@ class MPMesh{ void assemblyElm0(); template void assemblyVtx1(); - + template void setReconstructSlice(int order, MeshFieldType type); void reconstructSlices(); diff --git a/src/pmpo_MPMesh_assembly.hpp b/src/pmpo_MPMesh_assembly.hpp index c325a10..25899b9 100644 --- a/src/pmpo_MPMesh_assembly.hpp +++ b/src/pmpo_MPMesh_assembly.hpp @@ -98,34 +98,28 @@ void MPMesh::assemblyElm0() { pumipic::RecordTime("PolyMPO_Reconstruct_Elm0", timer.seconds()); } -template -void MPMesh::assemblyVtx1() { - + +void MPMesh::resetPreComputeFlag(){ + isPreComputed = false; +} + +void MPMesh::computeMatricesAndSolve(){ + //Mesh Information auto elm2VtxConn = p_mesh->getElm2VtxConn(); int numVtx = p_mesh->getNumVertices(); auto vtxCoords = p_mesh->getMeshField(); - //Mesh Field - constexpr MaterialPointSlice mpfIndex = meshFieldIndexToMPSlice; - const int numEntries = mpSliceToNumEntries(); - p_mesh->fillMeshField(numVtx, numEntries, 0.0); - auto meshField = p_mesh->getMeshField(); - //Dual Element Area for Regularization auto dual_triangle_area=p_mesh->getMeshField(); //Material Points - auto mpData = p_MPs->getData(); auto weight = p_MPs->getData(); auto mpPositions = p_MPs->getData(); //Matrix for each vertex Kokkos::View VtxMatrices("VtxMatrices", p_mesh->getNumVertices()); - //Reconstructed values - Kokkos::View reconVals("meshField", p_mesh->getNumVertices(), numEntries); - //Earth Radius double radius = 1.0; if(p_mesh->getGeomType() == geom_spherical_surf) @@ -141,22 +135,19 @@ void MPMesh::assemblyVtx1() { for(int i=0; iparallel_for(assemble, "assembly"); - //Solve Ax=b for each vertex and apply regularization + //Assemble matrix for each vertex and apply regularization Kokkos::View VtxCoeffs("VtxCoeffs", p_mesh->getNumVertices()); Kokkos::parallel_for("solving Ax=b", numVtx, KOKKOS_LAMBDA(const int vtx){ @@ -188,7 +179,8 @@ void MPMesh::assemblyVtx1() { break; } case 2:{ - double regParam=sqrt(EPSILON)*(VtxMatrices(vtx,0,0)+VtxMatrices(vtx,1,1)+VtxMatrices(vtx,2,2)+VtxMatrices(vtx,3,3)); + double regParam=sqrt(EPSILON)*(VtxMatrices(vtx,0,0)+VtxMatrices(vtx,1,1)+ + VtxMatrices(vtx,2,2)+VtxMatrices(vtx,3,3)); A_regularized.addToDiag(regParam); break; } @@ -212,7 +204,43 @@ void MPMesh::assemblyVtx1() { for (int i=0; iprecomputedVtxCoeffs = VtxCoeffs; +} + +template +void MPMesh::assemblyVtx1() { + + //If no reconstruction till now calculate the coeffs + if (!isPreComputed) { + computeMatricesAndSolve(); + isPreComputed=true; + } + auto VtxCoeffs=this->precomputedVtxCoeffs; + //Mesh Information + auto elm2VtxConn = p_mesh->getElm2VtxConn(); + int numVtx = p_mesh->getNumVertices(); + auto vtxCoords = p_mesh->getMeshField(); + + //Mesh Field + constexpr MaterialPointSlice mpfIndex = meshFieldIndexToMPSlice; + const int numEntries = mpSliceToNumEntries(); + p_mesh->fillMeshField(numVtx, numEntries, 0.0); + auto meshField = p_mesh->getMeshField(); + + //Material Points + auto mpData = p_MPs->getData(); + auto weight = p_MPs->getData(); + auto mpPositions = p_MPs->getData(); + + //Earth Radius + double radius = 1.0; + if(p_mesh->getGeomType() == geom_spherical_surf) + radius=p_mesh->getSphereRadius(); + + //Reconstructed values + Kokkos::View reconVals("meshField", p_mesh->getNumVertices(), numEntries); + //Reconstruct auto reconstruct = PS_LAMBDA(const int& elm, const int& mp, const int& mask) { if(mask) { //if material point is 'active'/'enabled' @@ -223,7 +251,7 @@ void MPMesh::assemblyVtx1() { double CoordDiffs[vec4d_nEntries] = {1, (vtxCoords(vID,0) - mpPositions(mp,0))/radius, (vtxCoords(vID,1) - mpPositions(mp,1))/radius, (vtxCoords(vID,2) - mpPositions(mp,2))/radius}; - + auto factor = w_vtx*(VtxCoeffs(vID,0) + VtxCoeffs(vID,1)*CoordDiffs[1] + VtxCoeffs(vID,2)*CoordDiffs[2] + VtxCoeffs(vID,3)*CoordDiffs[3]); @@ -236,7 +264,8 @@ void MPMesh::assemblyVtx1() { } }; p_MPs->parallel_for(reconstruct, "reconstruct"); - + + //Assign as a field Kokkos::parallel_for("assigning", numVtx, KOKKOS_LAMBDA(const int vtx){ for(int k=0; k Date: Tue, 25 Mar 2025 10:36:15 -0700 Subject: [PATCH 06/42] Multi process debugging --- src/pmpo_MPMesh.cpp | 32 +++++++++++++++++++++++++++++++- src/pmpo_MPMesh_assembly.hpp | 6 ++++-- src/pmpo_c.cpp | 33 ++++++++++++++++++++++++--------- 3 files changed, 59 insertions(+), 12 deletions(-) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index a5f709a..3c5fec3 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -127,7 +127,12 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ auto MPs2Elm = p_MPs->getData(); auto MPs2Proc = p_MPs->getData(); auto elm2Process = p_mesh->getElm2Process(); - + + Kokkos::parallel_for("countProcess", numElms, KOKKOS_LAMBDA(const int iElm){ + int pp_id=elm2Process(iElm); + printf("Mesh elm %d owning element %d \n", iElm, pp_id); + }); + if(printVTPIndex>=0) { printVTP_mesh(printVTPIndex); } @@ -324,15 +329,37 @@ bool getAnyIsMigrating(MaterialPoints* p_MPs, bool isMigrating) { } void MPMesh::push(){ + static int count=0; + std::cout<<__FUNCTION__<<" "<1) exit(1); Kokkos::Timer timer; p_mesh->computeRotLatLonIncr(); + + assert(cudaDeviceSynchronize() == cudaSuccess); + MPI_Barrier(MPI_COMM_WORLD); + printf("FooPush\n"); + sphericalInterpolation(*this); + assert(cudaDeviceSynchronize() == cudaSuccess); + MPI_Barrier(MPI_COMM_WORLD); + printf("FooPush \n"); + p_MPs->updateRotLatLonAndXYZ2Tgt(p_mesh->getSphereRadius()); // set Tgt_XYZ + assert(cudaDeviceSynchronize() == cudaSuccess); + MPI_Barrier(MPI_COMM_WORLD); + printf("FooPush\n"); + auto elm2Process = p_mesh->getElm2Process(); + assert(cudaDeviceSynchronize() == cudaSuccess); + MPI_Barrier(MPI_COMM_WORLD); + printf("FooPush \n"); bool anyIsMigrating = false; do { CVTTrackingElmCenterBased(); // move to Tgt_XYZ + assert(cudaDeviceSynchronize() == cudaSuccess); + printf("FooPushInside\n"); p_MPs->updateMPSlice(); // Tgt_XYZ becomes Cur_XYZ p_MPs->updateMPSlice(); // Tgt becomes Cur if (elm2Process.size() > 0) @@ -343,6 +370,9 @@ void MPMesh::push(){ reconstructSlices(); } while (anyIsMigrating); + assert(cudaDeviceSynchronize() == cudaSuccess); + MPI_Barrier(MPI_COMM_WORLD); + printf("FooPush\n"); pumipic::RecordTime("PolyMPO_push", timer.seconds()); } diff --git a/src/pmpo_MPMesh_assembly.hpp b/src/pmpo_MPMesh_assembly.hpp index 25899b9..69e02e6 100644 --- a/src/pmpo_MPMesh_assembly.hpp +++ b/src/pmpo_MPMesh_assembly.hpp @@ -104,7 +104,7 @@ void MPMesh::resetPreComputeFlag(){ } void MPMesh::computeMatricesAndSolve(){ - + Kokkos::Timer timer; //Mesh Information auto elm2VtxConn = p_mesh->getElm2VtxConn(); int numVtx = p_mesh->getNumVertices(); @@ -205,11 +205,12 @@ void MPMesh::computeMatricesAndSolve(){ VtxCoeffs(vtx,i)=coeff[i]; }); this->precomputedVtxCoeffs = VtxCoeffs; + pumipic::RecordTime("PolyMPO_Calculate_MLS_Coeff", timer.seconds()); } template void MPMesh::assemblyVtx1() { - + Kokkos::Timer timer; //If no reconstruction till now calculate the coeffs if (!isPreComputed) { computeMatricesAndSolve(); @@ -270,6 +271,7 @@ void MPMesh::assemblyVtx1() { for(int k=0; k diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index 67653b5..57a07ad 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -61,17 +61,19 @@ void polympo_setMPICommunicator_f(MPMesh_ptr p_mpmesh, MPI_Fint fcomm){ void polympo_createMPs_f(MPMesh_ptr p_mpmesh, const int numElms, - const int numMPs, // total number of MPs which is GREATER than or equal to number of active MPs + const int numMPs, // total nof of MPs which is >= no of active MPs int* mpsPerElm, const int* mp2Elm, const int* isMPActive) { checkMPMeshValid(p_mpmesh); - + std::cout<<__FUNCTION__<p_mesh; PMT_ALWAYS_ASSERT(!p_mesh->meshEditable()); PMT_ALWAYS_ASSERT(p_mesh->getNumElements() == numElms); + //Find the total no of MPs across all ranks + //And loop over all MPs and find the smallest element id associated across a MP int numActiveMPs = 0; int minElmID = numElms+1; for(int i = 0; i < numMPs; i++) { @@ -82,20 +84,27 @@ void polympo_createMPs_f(MPMesh_ptr p_mpmesh, } } } - //TODO do we care about empty ranks? check just in case... - PMT_ALWAYS_ASSERT(numActiveMPs>0); - - int firstElmWithMPs=-1; + long long globalNumActiveMPs = 0; + int globalMinElmID; + MPI_Allreduce(&numActiveMPs, &globalNumActiveMPs, 1, MPI_LONG_LONG_INT, MPI_SUM, MPI_COMM_WORLD); + MPI_Allreduce(&minElmID, &globalMinElmID, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD); + PMT_ALWAYS_ASSERT(globalNumActiveMPs>0); + + //Loop over all mesh elements 0,1,... and find the first element that has an associated MP + int firstElmWithMPs=numElms+1; for (int i=0; ip_MPs; ((polyMPO::MPMesh*)p_mpmesh)->p_MPs = new polyMPO::MaterialPoints(numElms, numActiveMPs, mpsPerElm_d, active_mp2Elm_d, active_mpIDs_d); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; p_MPs->setElmIDoffset(offset); + + assert(cudaDeviceSynchronize() == cudaSuccess); + MPI_Barrier(MPI_COMM_WORLD); + printf("Foo1\n"); + } void polympo_startRebuildMPs_f(MPMesh_ptr p_mpmesh, From 7be04f94d08b42554986d4ce7738eff801049647 Mon Sep 17 00:00:00 2001 From: dhyan1272 Date: Thu, 27 Mar 2025 12:41:35 -0700 Subject: [PATCH 07/42] More debugging --- src/pmpo_MPMesh.cpp | 8 ++++++-- src/pmpo_c.cpp | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index 3c5fec3..d213649 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -127,7 +127,11 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ auto MPs2Elm = p_MPs->getData(); auto MPs2Proc = p_MPs->getData(); auto elm2Process = p_mesh->getElm2Process(); - + + assert(cudaDeviceSynchronize() == cudaSuccess); + MPI_Barrier(MPI_COMM_WORLD); + printf("FooTracking \n"); + Kokkos::parallel_for("countProcess", numElms, KOKKOS_LAMBDA(const int iElm){ int pp_id=elm2Process(iElm); printf("Mesh elm %d owning element %d \n", iElm, pp_id); @@ -332,7 +336,7 @@ void MPMesh::push(){ static int count=0; std::cout<<__FUNCTION__<<" "<1) exit(1); + //if(count>1) exit(1); Kokkos::Timer timer; p_mesh->computeRotLatLonIncr(); diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index 57a07ad..406e88e 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -70,7 +70,7 @@ void polympo_createMPs_f(MPMesh_ptr p_mpmesh, //the mesh must be fixed/set before adding MPs auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; PMT_ALWAYS_ASSERT(!p_mesh->meshEditable()); - PMT_ALWAYS_ASSERT(p_mesh->getNumElements() == numElms); + //PMT_ALWAYS_ASSERT(p_mesh->getNumElements() == numElms); //Find the total no of MPs across all ranks //And loop over all MPs and find the smallest element id associated across a MP @@ -84,6 +84,7 @@ void polympo_createMPs_f(MPMesh_ptr p_mpmesh, } } } + printf("Num Active MPs and minElmId %d %d\n", numActiveMPs, minElmID); long long globalNumActiveMPs = 0; int globalMinElmID; MPI_Allreduce(&numActiveMPs, &globalNumActiveMPs, 1, MPI_LONG_LONG_INT, MPI_SUM, MPI_COMM_WORLD); @@ -98,6 +99,7 @@ void polympo_createMPs_f(MPMesh_ptr p_mpmesh, break; } } + printf("First elem with MP %d\n", firstElmWithMPs); int globalFirstElmWithMPs; MPI_Allreduce(&firstElmWithMPs, &globalFirstElmWithMPs, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD); From fefd5e58295fa96e10633f4b77ef87e8cd36f48f Mon Sep 17 00:00:00 2001 From: nathd2 Date: Mon, 31 Mar 2025 11:05:13 -0400 Subject: [PATCH 08/42] Added global ids to polyMPO from MPAS --- src/pmpo_MPMesh.cpp | 40 +++++++++++++-------------------- src/pmpo_c.cpp | 45 ++++++++++++++++++++++++------------- src/pmpo_c.h | 2 ++ src/pmpo_fortran.f90 | 15 +++++++++++++ src/pmpo_materialPoints.cpp | 10 ++++++--- src/pmpo_materialPoints.hpp | 2 +- src/pmpo_mesh.cpp | 6 +++++ src/pmpo_mesh.hpp | 6 ++++- 8 files changed, 81 insertions(+), 45 deletions(-) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index d213649..b282231 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -127,15 +127,21 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ auto MPs2Elm = p_MPs->getData(); auto MPs2Proc = p_MPs->getData(); auto elm2Process = p_mesh->getElm2Process(); - + auto elm2global = p_mesh->getElmGlobal(); + + + MPI_Comm comm = p_MPs->getMPIComm(); + int comm_rank; + MPI_Comm_rank(comm, &comm_rank); assert(cudaDeviceSynchronize() == cudaSuccess); MPI_Barrier(MPI_COMM_WORLD); printf("FooTracking \n"); - Kokkos::parallel_for("countProcess", numElms, KOKKOS_LAMBDA(const int iElm){ int pp_id=elm2Process(iElm); - printf("Mesh elm %d owning element %d \n", iElm, pp_id); + if(iElm<3 || iElm==1470 || iElm==1471 || iElm==1472 ) + printf("Mesh elm %d Process %d owning proc %d global %d \n", iElm, comm_rank, pp_id, elm2global(iElm)); }); + if(printVTPIndex>=0) { printVTP_mesh(printVTPIndex); @@ -333,51 +339,37 @@ bool getAnyIsMigrating(MaterialPoints* p_MPs, bool isMigrating) { } void MPMesh::push(){ + static int count=0; std::cout<<__FUNCTION__<<" "<1) exit(1); Kokkos::Timer timer; + p_mesh->computeRotLatLonIncr(); - assert(cudaDeviceSynchronize() == cudaSuccess); - MPI_Barrier(MPI_COMM_WORLD); - printf("FooPush\n"); - sphericalInterpolation(*this); - assert(cudaDeviceSynchronize() == cudaSuccess); - MPI_Barrier(MPI_COMM_WORLD); - printf("FooPush \n"); - + p_MPs->updateRotLatLonAndXYZ2Tgt(p_mesh->getSphereRadius()); // set Tgt_XYZ - assert(cudaDeviceSynchronize() == cudaSuccess); - MPI_Barrier(MPI_COMM_WORLD); - printf("FooPush\n"); - + auto elm2Process = p_mesh->getElm2Process(); - assert(cudaDeviceSynchronize() == cudaSuccess); - MPI_Barrier(MPI_COMM_WORLD); - printf("FooPush \n"); bool anyIsMigrating = false; do { CVTTrackingElmCenterBased(); // move to Tgt_XYZ assert(cudaDeviceSynchronize() == cudaSuccess); - printf("FooPushInside\n"); + printf("FooPushDoWhile\n"); p_MPs->updateMPSlice(); // Tgt_XYZ becomes Cur_XYZ p_MPs->updateMPSlice(); // Tgt becomes Cur if (elm2Process.size() > 0) anyIsMigrating = getAnyIsMigrating(p_MPs, p_MPs->migrate()); else p_MPs->rebuild(); //rebuild pumi-pic + printf("Is migrating %d \n", anyIsMigrating); p_MPs->updateMPElmID(); //update mpElm IDs slices reconstructSlices(); } while (anyIsMigrating); - assert(cudaDeviceSynchronize() == cudaSuccess); - MPI_Barrier(MPI_COMM_WORLD); - printf("FooPush\n"); - + pumipic::RecordTime("PolyMPO_push", timer.seconds()); } diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index 406e88e..b60a63a 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -75,34 +75,33 @@ void polympo_createMPs_f(MPMesh_ptr p_mpmesh, //Find the total no of MPs across all ranks //And loop over all MPs and find the smallest element id associated across a MP int numActiveMPs = 0; - int minElmID = numElms+1; + int minElmID = INT_MAX; for(int i = 0; i < numMPs; i++) { if(isMPActive[i] == MP_ACTIVE) { - if(mp2Elm[i] < minElmID) { + numActiveMPs++; + if(mp2Elm[i] < minElmID) minElmID = mp2Elm[i]; - numActiveMPs++; - } } } - printf("Num Active MPs and minElmId %d %d\n", numActiveMPs, minElmID); - long long globalNumActiveMPs = 0; + int globalNumActiveMPs = 0; int globalMinElmID; - MPI_Allreduce(&numActiveMPs, &globalNumActiveMPs, 1, MPI_LONG_LONG_INT, MPI_SUM, MPI_COMM_WORLD); - MPI_Allreduce(&minElmID, &globalMinElmID, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD); + MPI_Allreduce(&numActiveMPs, &globalNumActiveMPs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + MPI_Allreduce(&minElmID, &globalMinElmID, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD); PMT_ALWAYS_ASSERT(globalNumActiveMPs>0); //Loop over all mesh elements 0,1,... and find the first element that has an associated MP - int firstElmWithMPs=numElms+1; + int firstElmWithMPs=INT_MAX; for (int i=0; igetElmGlobal(); auto mpsPerElm_d = create_mirror_view_and_copy(mpsPerElm, numElms); auto active_mp2Elm_d = create_mirror_view_and_copy(active_mp2Elm.data(), numActiveMPs); @@ -130,19 +130,15 @@ void polympo_createMPs_f(MPMesh_ptr p_mpmesh, delete ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; ((polyMPO::MPMesh*)p_mpmesh)->p_MPs = - new polyMPO::MaterialPoints(numElms, numActiveMPs, mpsPerElm_d, active_mp2Elm_d, active_mpIDs_d); + new polyMPO::MaterialPoints(numElms, numActiveMPs, mpsPerElm_d, active_mp2Elm_d, active_mpIDs_d, elm2global); auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; p_MPs->setElmIDoffset(offset); - assert(cudaDeviceSynchronize() == cudaSuccess); - MPI_Barrier(MPI_COMM_WORLD); - printf("Foo1\n"); - } void polympo_startRebuildMPs_f(MPMesh_ptr p_mpmesh, - const int numMPs, // total number of MPs which is GREATER than or equal to number of active MPs + const int numMPs, // Total # MPs which is GREATER than or equal to number of active MPs const int* allMP2Elm, const int* addedMPMask) { checkMPMeshValid(p_mpmesh); @@ -1046,6 +1042,23 @@ void polympo_setOwningProc_f(MPMesh_ptr p_mpmesh, const int nCells, const int* a p_mesh->setOwningProc(owningProc); } +void polympo_setElmGlobal_f(MPMesh_ptr p_mpmesh, const int nCells, const int* array){ + checkMPMeshValid(p_mpmesh); + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + PMT_ALWAYS_ASSERT(p_mesh->meshEditable()); + Kokkos::View arrayHost("arrayHost", nCells); + for (int i = 0; i < nCells; i++) { + arrayHost(i) = array[i] - 1; // Decrease each value by 1 + } + //check the size + PMT_ALWAYS_ASSERT(nCells == p_mesh->getNumElements()); + + Kokkos::View elmGlobal("elmGlobal",nCells); + Kokkos::deep_copy(elmGlobal, arrayHost); + p_mesh->setElmGlobal(elmGlobal); +} + + void polympo_enableTiming_f(){ pumipic::EnableTiming(); } diff --git a/src/pmpo_c.h b/src/pmpo_c.h index 174f862..f62ffdc 100644 --- a/src/pmpo_c.h +++ b/src/pmpo_c.h @@ -57,6 +57,8 @@ void polympo_setMeshNumEdgesPerElm_f(MPMesh_ptr p_mpmesh, const int nCells, cons void polympo_setMeshElm2VtxConn_f(MPMesh_ptr p_mpmesh, const int maxEdges, const int nCells, const int* array); void polympo_setMeshElm2ElmConn_f(MPMesh_ptr p_mpmesh, const int maxEdges, const int nCells, const int* array); void polympo_setOwningProc_f(MPMesh_ptr p_mpmesh, const int nCells, const int* array); +void polympo_setElmGlobal_f(MPMesh_ptr p_mpmesh, const int nCells, const int* array); + //Mesh fields int polympo_getMeshFVtxType_f(); diff --git a/src/pmpo_fortran.f90 b/src/pmpo_fortran.f90 index cf013ca..f386058 100644 --- a/src/pmpo_fortran.f90 +++ b/src/pmpo_fortran.f90 @@ -698,6 +698,21 @@ subroutine polympo_setOwningProc(mpMesh, nCells, array) & integer(c_int), value :: nCells type(c_ptr), intent(in), value :: array end subroutine + !--------------------------------------------------------------------------- + !> @brief set the owning process array + !> @param mpmesh(in/out) MPMesh object + !> @param nCells(in) number of cells + !> @param array(in) input mesh cell to process array + !--------------------------------------------------------------------------- + subroutine polympo_setElmGlobal(mpMesh, nCells, array) & + bind(C, NAME='polympo_setElmGlobal_f') + use :: iso_c_binding + type(c_ptr), value :: mpMesh + integer(c_int), value :: nCells + type(c_ptr), intent(in), value :: array + end subroutine + + !--------------------------------------------------------------------------- !> @brief calculate the MPs from given mesh vertices rotational latitude !> longitude, update the MP slices diff --git a/src/pmpo_materialPoints.cpp b/src/pmpo_materialPoints.cpp index c20fe00..5e4cdae 100644 --- a/src/pmpo_materialPoints.cpp +++ b/src/pmpo_materialPoints.cpp @@ -40,8 +40,12 @@ PS* createDPS(int numElms, int numMPs, MPSView positions, IntVi return dps; } -PS* createDPS(int numElms, int numMPs, IntView mpsPerElm, IntView mp2elm, IntView mpAppID) { +PS* createDPS(int numElms, int numMPs, IntView mpsPerElm, IntView mp2elm, IntView mpAppID, IntView elm2global) { PS::kkGidView elmGids("elementGlobalIds", numElms); //TODO - initialize this to [0..numElms) + Kokkos::parallel_for("setGids", numElms, KOKKOS_LAMBDA(const int elm){ + elmGids(elm) = elm2global(elm); + }); + auto mpInfo = createInternalMemberViews(numMPs, mp2elm, mpAppID); Kokkos::TeamPolicy policy(numElms,Kokkos::AUTO); auto dps = new DPS(policy, numElms, numMPs, mpsPerElm, elmGids, mp2elm, mpInfo); @@ -57,8 +61,8 @@ MaterialPoints::MaterialPoints(int numElms, int numMPs, MPSView operating_mode = MP_RELEASE; }; -MaterialPoints::MaterialPoints(int numElms, int numMPs, IntView mpsPerElm, IntView mp2elm, IntView mpAppID) { - MPs = createDPS(numElms, numMPs, mpsPerElm, mp2elm, mpAppID); +MaterialPoints::MaterialPoints(int numElms, int numMPs, IntView mpsPerElm, IntView mp2elm, IntView mpAppID, IntView elm2global){ + MPs = createDPS(numElms, numMPs, mpsPerElm, mp2elm, mpAppID, elm2global); updateMaxAppID(); operating_mode = MP_RELEASE; }; diff --git a/src/pmpo_materialPoints.hpp b/src/pmpo_materialPoints.hpp index 99d2964..1701985 100644 --- a/src/pmpo_materialPoints.hpp +++ b/src/pmpo_materialPoints.hpp @@ -130,7 +130,7 @@ class MaterialPoints { public: MaterialPoints() : MPs(nullptr) {}; MaterialPoints(int numElms, int numMPs, MPSView positions, IntView mpsPerElm, IntView mp2elm); - MaterialPoints(int numElms, int numMPs, IntView mpsPerElm, IntView mp2elm, IntView mpAppID); + MaterialPoints(int numElms, int numMPs, IntView mpsPerElm, IntView mp2elm, IntView mpAppID, IntView elm2global); ~MaterialPoints(); void rebuild(IntView addedMP2elm, IntView addedMPAppID); diff --git a/src/pmpo_mesh.cpp b/src/pmpo_mesh.cpp index 9232ad1..57b838f 100644 --- a/src/pmpo_mesh.cpp +++ b/src/pmpo_mesh.cpp @@ -77,5 +77,11 @@ namespace polyMPO{ IntView Mesh::getElm2Process() { return owningProc_; } + + IntView Mesh::getElmGlobal() { + return globalElm_; + } + + } // namespace polyMPO diff --git a/src/pmpo_mesh.hpp b/src/pmpo_mesh.hpp index 2058e13..0819da2 100644 --- a/src/pmpo_mesh.hpp +++ b/src/pmpo_mesh.hpp @@ -88,7 +88,7 @@ class Mesh { IntVtx2ElmView elm2VtxConn_; IntElm2ElmView elm2ElmConn_; IntView owningProc_; - + IntView globalElm_; //start of meshFields MeshFView vtxCoords_; MeshFView vtxRotLat_; @@ -163,6 +163,10 @@ class Mesh { void setOwningProc(IntView owningProc) {PMT_ALWAYS_ASSERT(meshEdit_); owningProc_ = owningProc; } + void setElmGlobal(IntView globalElm) {PMT_ALWAYS_ASSERT(meshEdit_); + globalElm_ = globalElm; } + IntView getElmGlobal(); + void computeRotLatLonIncr(); }; From 8bc07803aebf619050c46c59086d727cb7b15e9c Mon Sep 17 00:00:00 2001 From: nathd2 Date: Tue, 1 Apr 2025 15:20:35 -0400 Subject: [PATCH 09/42] Running on multiple processors --- src/pmpo_MPMesh.cpp | 22 +++++++++------------- src/pmpo_c.cpp | 5 ++--- src/pmpo_materialPoints.cpp | 26 ++++++++++++++++++++++---- src/pmpo_materialPoints.hpp | 3 ++- 4 files changed, 35 insertions(+), 21 deletions(-) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index b282231..f2f0929 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -129,20 +129,13 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ auto elm2Process = p_mesh->getElm2Process(); auto elm2global = p_mesh->getElmGlobal(); - MPI_Comm comm = p_MPs->getMPIComm(); int comm_rank; MPI_Comm_rank(comm, &comm_rank); - assert(cudaDeviceSynchronize() == cudaSuccess); - MPI_Barrier(MPI_COMM_WORLD); - printf("FooTracking \n"); Kokkos::parallel_for("countProcess", numElms, KOKKOS_LAMBDA(const int iElm){ int pp_id=elm2Process(iElm); - if(iElm<3 || iElm==1470 || iElm==1471 || iElm==1472 ) - printf("Mesh elm %d Process %d owning proc %d global %d \n", iElm, comm_rank, pp_id, elm2global(iElm)); }); - if(printVTPIndex>=0) { printVTP_mesh(printVTPIndex); } @@ -341,8 +334,9 @@ bool getAnyIsMigrating(MaterialPoints* p_MPs, bool isMigrating) { void MPMesh::push(){ static int count=0; - std::cout<<__FUNCTION__<<" "<computeRotLatLonIncr(); @@ -357,14 +351,16 @@ void MPMesh::push(){ do { CVTTrackingElmCenterBased(); // move to Tgt_XYZ assert(cudaDeviceSynchronize() == cudaSuccess); - printf("FooPushDoWhile\n"); p_MPs->updateMPSlice(); // Tgt_XYZ becomes Cur_XYZ p_MPs->updateMPSlice(); // Tgt becomes Cur - if (elm2Process.size() > 0) - anyIsMigrating = getAnyIsMigrating(p_MPs, p_MPs->migrate()); + + bool anyIsMigrating = getAnyIsMigrating(p_MPs, p_MPs->check_migrate()); + + if(anyIsMigrating) + p_MPs->migrate(); else - p_MPs->rebuild(); //rebuild pumi-pic - printf("Is migrating %d \n", anyIsMigrating); + p_MPs->rebuild(); + p_MPs->updateMPElmID(); //update mpElm IDs slices reconstructSlices(); } diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index b60a63a..19f1307 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -66,11 +66,10 @@ void polympo_createMPs_f(MPMesh_ptr p_mpmesh, const int* mp2Elm, const int* isMPActive) { checkMPMeshValid(p_mpmesh); - std::cout<<__FUNCTION__<p_mesh; PMT_ALWAYS_ASSERT(!p_mesh->meshEditable()); - //PMT_ALWAYS_ASSERT(p_mesh->getNumElements() == numElms); + PMT_ALWAYS_ASSERT(p_mesh->getNumElements() == numElms); //Find the total no of MPs across all ranks //And loop over all MPs and find the smallest element id associated across a MP @@ -100,7 +99,7 @@ void polympo_createMPs_f(MPMesh_ptr p_mpmesh, int globalFirstElmWithMPs; MPI_Allreduce(&firstElmWithMPs, &globalFirstElmWithMPs, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD); - printf("With a MP, globally smallest mesh elm %d and first elm %d \n", globalMinElmID, globalFirstElmWithMPs); + //printf("With a MP, globally smallest mesh elm %d and first elm %d \n", globalMinElmID, globalFirstElmWithMPs); int offset = -1; if(globalMinElmID-globalFirstElmWithMPs==1) { diff --git a/src/pmpo_materialPoints.cpp b/src/pmpo_materialPoints.cpp index 5e4cdae..1d99c95 100644 --- a/src/pmpo_materialPoints.cpp +++ b/src/pmpo_materialPoints.cpp @@ -113,14 +113,34 @@ void MaterialPoints::setMPIComm(MPI_Comm comm) { mpi_comm = comm; } -bool MaterialPoints::migrate() { +bool MaterialPoints::check_migrate(){ + Kokkos::Timer timer; + auto MPs2Elm = getData(); + auto MPs2Proc = getData(); + + IntView isMigrating("isMigrating", 1); + + int rank; + MPI_Comm_rank(mpi_comm, &rank); + auto setMigrationFields = PS_LAMBDA(const int& e, const int& mp, const bool& mask) { + if (mask) { + if (MPs2Proc(mp) != rank) isMigrating(0) = 1; + } + }; + parallel_for(setMigrationFields, "setMigrationFields"); + if (getOpMode() == polyMPO::MP_DEBUG) + printf("Material point check migration: %f\n", timer.seconds()); + pumipic::RecordTime("PolyMPO_check_migrate", timer.seconds()); + return pumipic::getLastValue(isMigrating) > 0; +} + +void MaterialPoints::migrate() { Kokkos::Timer timer; auto MPs2Elm = getData(); auto MPs2Proc = getData(); IntView new_elem("new_elem", MPs->capacity()); IntView new_process("new_process", MPs->capacity()); - IntView isMigrating("isMigrating", 1); int rank; MPI_Comm_rank(mpi_comm, &rank); @@ -128,7 +148,6 @@ bool MaterialPoints::migrate() { if (mask) { new_elem(mp) = MPs2Elm(mp); new_process(mp) = MPs2Proc(mp); - if (new_process(mp) != rank) isMigrating(0) = 1; } }; parallel_for(setMigrationFields, "setMigrationFields"); @@ -137,7 +156,6 @@ bool MaterialPoints::migrate() { if (getOpMode() == polyMPO::MP_DEBUG) printf("Material point migration: %f\n", timer.seconds()); pumipic::RecordTime("PolyMPO_migrate", timer.seconds()); - return pumipic::getLastValue(isMigrating) > 0; } bool MaterialPoints::rebuildOngoing() { return rebuildFields.ongoing; } diff --git a/src/pmpo_materialPoints.hpp b/src/pmpo_materialPoints.hpp index 1701985..7511a1d 100644 --- a/src/pmpo_materialPoints.hpp +++ b/src/pmpo_materialPoints.hpp @@ -138,7 +138,8 @@ class MaterialPoints { void finishRebuild(); bool rebuildOngoing(); - bool migrate(); + bool check_migrate(); + void migrate(); MPI_Comm getMPIComm(); void setMPIComm(MPI_Comm comm); From e004bc53c13e9cc617cfb504ca069ecd906b03f7 Mon Sep 17 00:00:00 2001 From: nathd2 Date: Mon, 7 Apr 2025 11:39:50 -0400 Subject: [PATCH 10/42] Vtp files issue for multi-process --- src/pmpo_MPMesh.cpp | 50 ++++++++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index f2f0929..db27d06 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -135,16 +135,21 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ Kokkos::parallel_for("countProcess", numElms, KOKKOS_LAMBDA(const int iElm){ int pp_id=elm2Process(iElm); }); - - if(printVTPIndex>=0) { - printVTP_mesh(printVTPIndex); + + //Since Mesh is static print pnly for 1 time step + if(printVTPIndex==0) { + printVTP_mesh(comm_rank); } + + assert(cudaDeviceSynchronize()==cudaSuccess); + MPI_Barrier(MPI_COMM_WORLD); Vec3dView history("positionHistory",numMPs); Vec3dView resultLeft("positionResult",numMPs); Vec3dView resultRight("positionResult",numMPs); Vec3dView mpTgtPosArray("positionTarget",numMPs); - + Kokkos::View counter("counter",1); + auto CVTElmCalc = PS_LAMBDA(const int& elm, const int& mp, const int&mask){ Vec3d MP(mpPositions(mp,0),mpPositions(mp,1),mpPositions(mp,2)); if(mask){ @@ -163,7 +168,7 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ for(int i=1; i<=numConnElms; i++){ int elmID = elm2ElmConn(iElm,i)-1; - //New delta + //New delta Vec3d center(elmCenter(elmID, 0), elmCenter(elmID, 1), elmCenter(elmID, 2)); delta = MPnew - center; @@ -183,7 +188,9 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ iElm = closestElm; } } - if(printVTPIndex>=0){ + + if(printVTPIndex>=0 && numMPs>0){ + //printf("Rank %d mp %d counter %d \n", comm_rank, mp, counter); double d1 = dx[0]; double d2 = dx[2]; double d3 = dx[3]; @@ -195,29 +202,36 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ Vec3d shift = dx.cross(r) * ((1.0-0.7)*dx.magnitude()/(dx.cross(r)).magnitude()); Vec3d MPLeft = MParrow + shift; Vec3d MPRight = MParrow - shift; - history(mp) = MP; - resultLeft(mp) = MPLeft; - resultRight(mp) = MPRight; - mpTgtPosArray(mp) = MPnew; + auto xx=Kokkos::atomic_fetch_add(&counter(0), 1); + history(xx) = MP; + resultLeft(xx) = MPLeft; + resultRight(xx) = MPRight; + mpTgtPosArray(xx) = MPnew; } } }; p_MPs->parallel_for(CVTElmCalc,"CVTTrackingElmCenterBasedCalc"); - if(printVTPIndex>=0){ + assert(cudaDeviceSynchronize()==cudaSuccess); + MPI_Barrier(MPI_COMM_WORLD); + printf("After Tracking \n"); + + if(printVTPIndex>=0 && numMPs>0){ Vec3dView::HostMirror h_history = Kokkos::create_mirror_view(history); Vec3dView::HostMirror h_resultLeft = Kokkos::create_mirror_view(resultLeft); Vec3dView::HostMirror h_resultRight = Kokkos::create_mirror_view(resultRight); Vec3dView::HostMirror h_mpTgtPos = Kokkos::create_mirror_view(mpTgtPosArray); + Kokkos::View::HostMirror h_counter = Kokkos::create_mirror_view(counter); Kokkos::deep_copy(h_history, history); Kokkos::deep_copy(h_resultLeft, resultLeft); Kokkos::deep_copy(h_resultRight, resultRight); Kokkos::deep_copy(h_mpTgtPos, mpTgtPosArray); - + Kokkos::deep_copy(h_counter, counter); + printf("Host counter value: %d\n", h_counter(0)); // printVTP file char* fileOutput = (char *)malloc(sizeof(char) * 256); - sprintf(fileOutput, "polyMPOCVTTrackingElmCenter_MPtracks_%d.vtp", printVTPIndex); + sprintf(fileOutput, "polyMPOCVTTrackingElmCenter_MPtracks_%d_%d.vtp", comm_rank, printVTPIndex); FILE * pFile = fopen(fileOutput,"w"); free(fileOutput); fprintf(pFile, "\n \n \n \n \n",numMPs*4,numMPs*2); @@ -239,6 +253,11 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ fprintf(pFile," \n \n \n \n\n"); fclose(pFile); } + assert(cudaDeviceSynchronize()==cudaSuccess); + MPI_Barrier(MPI_COMM_WORLD); + printf("After printing particle paths \n"); + + pumipic::RecordTime("PolyMPO_CVTTrackingElmCenterBased", timer.seconds()); } @@ -335,7 +354,6 @@ void MPMesh::push(){ static int count=0; std::cout<<"Push"<<" "<updateMPSlice(); // Tgt_XYZ becomes Cur_XYZ p_MPs->updateMPSlice(); // Tgt becomes Cur @@ -365,7 +383,7 @@ void MPMesh::push(){ reconstructSlices(); } while (anyIsMigrating); - + count ++; pumipic::RecordTime("PolyMPO_push", timer.seconds()); } From 4274cd68a8245c71607f5b03a420c27bbc780d05 Mon Sep 17 00:00:00 2001 From: nathd2 Date: Wed, 9 Apr 2025 22:19:34 -0400 Subject: [PATCH 11/42] Towards multi Process --- src/pmpo_MPMesh.cpp | 9 --------- src/pmpo_c.cpp | 9 +++++++++ src/pmpo_c.h | 4 ++++ src/pmpo_defines.h | 1 + src/pmpo_fortran.f90 | 22 ++++++++++++++++++++++ src/pmpo_materialPoints.cpp | 30 ++++++++++++++++++++++++++++++ 6 files changed, 66 insertions(+), 9 deletions(-) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index db27d06..0509043 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -212,10 +212,6 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ }; p_MPs->parallel_for(CVTElmCalc,"CVTTrackingElmCenterBasedCalc"); - assert(cudaDeviceSynchronize()==cudaSuccess); - MPI_Barrier(MPI_COMM_WORLD); - printf("After Tracking \n"); - if(printVTPIndex>=0 && numMPs>0){ Vec3dView::HostMirror h_history = Kokkos::create_mirror_view(history); Vec3dView::HostMirror h_resultLeft = Kokkos::create_mirror_view(resultLeft); @@ -228,7 +224,6 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ Kokkos::deep_copy(h_resultRight, resultRight); Kokkos::deep_copy(h_mpTgtPos, mpTgtPosArray); Kokkos::deep_copy(h_counter, counter); - printf("Host counter value: %d\n", h_counter(0)); // printVTP file char* fileOutput = (char *)malloc(sizeof(char) * 256); sprintf(fileOutput, "polyMPOCVTTrackingElmCenter_MPtracks_%d_%d.vtp", comm_rank, printVTPIndex); @@ -253,10 +248,6 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ fprintf(pFile," \n \n \n \n\n"); fclose(pFile); } - assert(cudaDeviceSynchronize()==cudaSuccess); - MPI_Barrier(MPI_COMM_WORLD); - printf("After printing particle paths \n"); - pumipic::RecordTime("PolyMPO_CVTTrackingElmCenterBased", timer.seconds()); } diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index 19f1307..1f9ddea 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -212,6 +212,15 @@ void polympo_setAppIDFunc_f(MPMesh_ptr p_mpmesh, IntVoidFunc getNext, void* appI p_MPs->setAppIDFunc(getNextAppID); } +void polympo_setMPASAppIDFunc_f(MPMesh_ptr p_mpmesh, VoidVoidFunc getMPASAppID, void* arg1, void*arg2, void*arg3, + void* arg4, void* arg5, void*arg6) { + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + std::function polyMPO_getMPASAppID = [getMPASAppID, arg1, arg2, arg3, arg4, arg5, arg6]() + {getMPASAppID(arg1, arg2, arg3, arg4, arg5, arg6); return *(static_cast(arg5));}; + p_MPs->setAppIDFunc(polyMPO_getMPASAppID); +} + void polympo_getMPCurElmID_f(MPMesh_ptr p_mpmesh, const int numMPs, int* elmIDs){ diff --git a/src/pmpo_c.h b/src/pmpo_c.h index f62ffdc..d749745 100644 --- a/src/pmpo_c.h +++ b/src/pmpo_c.h @@ -22,7 +22,11 @@ void polympo_setMPICommunicator_f(MPMesh_ptr p_mpmesh, MPI_Fint fcomm); void polympo_createMPs_f(MPMesh_ptr p_mpmesh, const int numElms, const int numMPs, int* mpsPerElm, const int* mp2Elm, const int* isMPActive); void polympo_startRebuildMPs_f(MPMesh_ptr p_mpmesh, const int numMPs, const int* allTgtMpElmIn, const int* addedMPMask); void polympo_finishRebuildMPs_f(MPMesh_ptr p_mpmesh); + void polympo_setAppIDFunc_f(MPMesh_ptr p_mpmesh, IntVoidFunc getNext, void* appIDs); +void polympo_setMPASAppIDFunc_f(MPMesh_ptr p_mpmesh, VoidVoidFunc getMPASAppID, void* arg1, void* arg2, void*arg3, + void* arg4, void*arg5, void* arg6); + void polympo_getMPCurElmID_f(MPMesh_ptr p_mpmesh, const int numMPs, int* elmIDs); void polympo_setMPLatLonRotatedFlag_f(MPMesh_ptr p_mpmesh, const int isRotateFlag); diff --git a/src/pmpo_defines.h b/src/pmpo_defines.h index 34857b9..3bb8898 100644 --- a/src/pmpo_defines.h +++ b/src/pmpo_defines.h @@ -8,6 +8,7 @@ typedef void* MPMesh_ptr; //Function that receives void* and returns an int typedef int (*IntVoidFunc)(void*); +typedef void (*VoidVoidFunc)(void*, void*, void*, void*, void*, void*); using space_t = Kokkos::DefaultExecutionSpace::memory_space; diff --git a/src/pmpo_fortran.f90 b/src/pmpo_fortran.f90 index f386058..1c9f473 100644 --- a/src/pmpo_fortran.f90 +++ b/src/pmpo_fortran.f90 @@ -110,6 +110,28 @@ subroutine polympo_setAppIDFunc(mpMesh, getNext, appIDs) & type(c_funptr), value :: getNext type(c_ptr), value :: appIDs end subroutine + + !--------------------------------------------------------------------------- + !> @brief Stores pointer to appID data structure and a function to retrieve them used in migration + !> @param mpmesh(in/out) MPMesh object + !> @param getNext(in) Pointer to function that returns next App IDs + !> @param appIDs(in) Pointer to opaque data application data structure (that may contain all available app IDs) + !--------------------------------------------------------------------------- + subroutine polympo_setMPASAppIDFunc(mpMesh, getMPASAppID, & + arg1, arg2, arg3, arg4, arg5, arg6) & + bind(C, NAME='polympo_setMPASAppIDFunc_f') + use :: iso_c_binding + type(c_ptr), value :: mpMesh + type(c_funptr), value :: getMPASAppID + type(c_ptr), value :: arg1 + type(c_ptr), value :: arg2 + type(c_ptr), value :: arg3 + type(c_ptr), value :: arg4 + type(c_ptr), value :: arg5 + type(c_ptr), value :: arg6 + end subroutine + + !--------------------------------------------------------------------------- !> @brief get the current element ID MP array from a polympo array !> @param mpmesh(in/out) MPMesh object diff --git a/src/pmpo_materialPoints.cpp b/src/pmpo_materialPoints.cpp index 1d99c95..8b7f7a6 100644 --- a/src/pmpo_materialPoints.cpp +++ b/src/pmpo_materialPoints.cpp @@ -138,6 +138,7 @@ void MaterialPoints::migrate() { Kokkos::Timer timer; auto MPs2Elm = getData(); auto MPs2Proc = getData(); + auto mpAppID = getData(); IntView new_elem("new_elem", MPs->capacity()); IntView new_process("new_process", MPs->capacity()); @@ -148,11 +149,40 @@ void MaterialPoints::migrate() { if (mask) { new_elem(mp) = MPs2Elm(mp); new_process(mp) = MPs2Proc(mp); + if(rank!=new_process(mp)){ + mpAppID(mp)=-1; + printf("Particle migrated and so its AppID is -1\n"); + } } }; parallel_for(setMigrationFields, "setMigrationFields"); MPs->migrate(new_elem, new_process); + //Since rebuilt + mpAppID = getData(); + //Count MPs that have -1 appID, so that we can count no of MPs received + Kokkos::View numReceivedMPs("numReceivedMPs", 1); + Kokkos::deep_copy(numReceivedMPs, 0); + auto countnewMPs = PS_LAMBDA(const int& e, const int& mp, const bool& mask) { + if(mask){ + if (mpAppID(mp) == -1) + Kokkos::atomic_add(&numReceivedMPs(0), 1); + } + }; + parallel_for(countnewMPs, "countReceivedPtcls"); + Kokkos::fence(); + + auto numReceivedMPs_host = Kokkos::create_mirror_view(numReceivedMPs); + Kokkos::deep_copy(numReceivedMPs_host, numReceivedMPs); + if(numReceivedMPs_host(0)) + std::cout <<"Rank "< added_mpIDs(numReceivedMPs_host(0)); + for(int i=0; i Date: Thu, 10 Apr 2025 12:17:05 -0400 Subject: [PATCH 12/42] One more way of trying MPAS AppIDs --- src/pmpo_c.cpp | 6 +++--- src/pmpo_c.h | 2 +- src/pmpo_defines.h | 2 +- src/pmpo_fortran.f90 | 6 ++---- src/pmpo_materialPoints.cpp | 2 +- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index 1f9ddea..c705493 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -213,11 +213,11 @@ void polympo_setAppIDFunc_f(MPMesh_ptr p_mpmesh, IntVoidFunc getNext, void* appI } void polympo_setMPASAppIDFunc_f(MPMesh_ptr p_mpmesh, VoidVoidFunc getMPASAppID, void* arg1, void*arg2, void*arg3, - void* arg4, void* arg5, void*arg6) { + const int arg4) { checkMPMeshValid(p_mpmesh); auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - std::function polyMPO_getMPASAppID = [getMPASAppID, arg1, arg2, arg3, arg4, arg5, arg6]() - {getMPASAppID(arg1, arg2, arg3, arg4, arg5, arg6); return *(static_cast(arg5));}; + std::function polyMPO_getMPASAppID = [getMPASAppID, arg1, arg2, arg3, arg4]() + { int iParticleNew; getMPASAppID(arg1, arg2, arg3, arg4, iParticleNew); return iParticleNew;}; p_MPs->setAppIDFunc(polyMPO_getMPASAppID); } diff --git a/src/pmpo_c.h b/src/pmpo_c.h index d749745..be83a83 100644 --- a/src/pmpo_c.h +++ b/src/pmpo_c.h @@ -25,7 +25,7 @@ void polympo_finishRebuildMPs_f(MPMesh_ptr p_mpmesh); void polympo_setAppIDFunc_f(MPMesh_ptr p_mpmesh, IntVoidFunc getNext, void* appIDs); void polympo_setMPASAppIDFunc_f(MPMesh_ptr p_mpmesh, VoidVoidFunc getMPASAppID, void* arg1, void* arg2, void*arg3, - void* arg4, void*arg5, void* arg6); + const int arg4); void polympo_getMPCurElmID_f(MPMesh_ptr p_mpmesh, const int numMPs, int* elmIDs); void polympo_setMPLatLonRotatedFlag_f(MPMesh_ptr p_mpmesh, const int isRotateFlag); diff --git a/src/pmpo_defines.h b/src/pmpo_defines.h index 3bb8898..f4abcb0 100644 --- a/src/pmpo_defines.h +++ b/src/pmpo_defines.h @@ -8,7 +8,7 @@ typedef void* MPMesh_ptr; //Function that receives void* and returns an int typedef int (*IntVoidFunc)(void*); -typedef void (*VoidVoidFunc)(void*, void*, void*, void*, void*, void*); +typedef void (*VoidVoidFunc)(void*, void*, void*, const int, int); using space_t = Kokkos::DefaultExecutionSpace::memory_space; diff --git a/src/pmpo_fortran.f90 b/src/pmpo_fortran.f90 index 1c9f473..646a07d 100644 --- a/src/pmpo_fortran.f90 +++ b/src/pmpo_fortran.f90 @@ -118,7 +118,7 @@ subroutine polympo_setAppIDFunc(mpMesh, getNext, appIDs) & !> @param appIDs(in) Pointer to opaque data application data structure (that may contain all available app IDs) !--------------------------------------------------------------------------- subroutine polympo_setMPASAppIDFunc(mpMesh, getMPASAppID, & - arg1, arg2, arg3, arg4, arg5, arg6) & + arg1, arg2, arg3, arg4) & bind(C, NAME='polympo_setMPASAppIDFunc_f') use :: iso_c_binding type(c_ptr), value :: mpMesh @@ -126,9 +126,7 @@ subroutine polympo_setMPASAppIDFunc(mpMesh, getMPASAppID, & type(c_ptr), value :: arg1 type(c_ptr), value :: arg2 type(c_ptr), value :: arg3 - type(c_ptr), value :: arg4 - type(c_ptr), value :: arg5 - type(c_ptr), value :: arg6 + integer(c_int), intent(in), value :: arg4 end subroutine diff --git a/src/pmpo_materialPoints.cpp b/src/pmpo_materialPoints.cpp index 8b7f7a6..394163e 100644 --- a/src/pmpo_materialPoints.cpp +++ b/src/pmpo_materialPoints.cpp @@ -151,7 +151,7 @@ void MaterialPoints::migrate() { new_process(mp) = MPs2Proc(mp); if(rank!=new_process(mp)){ mpAppID(mp)=-1; - printf("Particle migrated and so its AppID is -1\n"); + printf("Particle migrated and so its AppID is -1\n"); } } }; From c52bfa73167516379b7a10ac949916c1f55de011 Mon Sep 17 00:00:00 2001 From: nathd2 Date: Fri, 11 Apr 2025 10:54:44 -0400 Subject: [PATCH 13/42] MPASAppID now takes cell no as input --- src/pmpo_c.cpp | 6 +++--- src/pmpo_defines.h | 2 +- src/pmpo_materialPoints.cpp | 27 +++++++++++++++++++++------ src/pmpo_materialPoints.hpp | 12 +++++++++--- 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index c705493..6616f9e 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -209,15 +209,15 @@ void polympo_setAppIDFunc_f(MPMesh_ptr p_mpmesh, IntVoidFunc getNext, void* appI checkMPMeshValid(p_mpmesh); auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; polyMPO::IntFunc getNextAppID = [getNext, appIDs]() { return getNext(appIDs); }; - p_MPs->setAppIDFunc(getNextAppID); + //p_MPs->setAppIDFunc(getNextAppID); } void polympo_setMPASAppIDFunc_f(MPMesh_ptr p_mpmesh, VoidVoidFunc getMPASAppID, void* arg1, void*arg2, void*arg3, const int arg4) { checkMPMeshValid(p_mpmesh); auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - std::function polyMPO_getMPASAppID = [getMPASAppID, arg1, arg2, arg3, arg4]() - { int iParticleNew; getMPASAppID(arg1, arg2, arg3, arg4, iParticleNew); return iParticleNew;}; + std::function polyMPO_getMPASAppID = [getMPASAppID, arg1, arg2, arg3, arg4](int iCell) + { int iParticleNew; getMPASAppID(arg1, arg2, arg3, arg4, iCell, iParticleNew); return iParticleNew;}; p_MPs->setAppIDFunc(polyMPO_getMPASAppID); } diff --git a/src/pmpo_defines.h b/src/pmpo_defines.h index f4abcb0..21a7d49 100644 --- a/src/pmpo_defines.h +++ b/src/pmpo_defines.h @@ -8,7 +8,7 @@ typedef void* MPMesh_ptr; //Function that receives void* and returns an int typedef int (*IntVoidFunc)(void*); -typedef void (*VoidVoidFunc)(void*, void*, void*, const int, int); +typedef void (*VoidVoidFunc)(void*, void*, void*, const int, int, int); using space_t = Kokkos::DefaultExecutionSpace::memory_space; diff --git a/src/pmpo_materialPoints.cpp b/src/pmpo_materialPoints.cpp index 394163e..9bea799 100644 --- a/src/pmpo_materialPoints.cpp +++ b/src/pmpo_materialPoints.cpp @@ -158,7 +158,7 @@ void MaterialPoints::migrate() { parallel_for(setMigrationFields, "setMigrationFields"); MPs->migrate(new_elem, new_process); - //Since rebuilt + //SINCE REBUILT, mpAppID needs to be be recalled mpAppID = getData(); //Count MPs that have -1 appID, so that we can count no of MPs received Kokkos::View numReceivedMPs("numReceivedMPs", 1); @@ -171,16 +171,31 @@ void MaterialPoints::migrate() { }; parallel_for(countnewMPs, "countReceivedPtcls"); Kokkos::fence(); - + //Bring them to CPU and print particles received auto numReceivedMPs_host = Kokkos::create_mirror_view(numReceivedMPs); Kokkos::deep_copy(numReceivedMPs_host, numReceivedMPs); if(numReceivedMPs_host(0)) std::cout <<"Rank "< added_mpIDs(numReceivedMPs_host(0)); + //Another array that contains element id of the the MPs that have migrated + Kokkos::View receivedMPs2Elm("ReceivedMPs2Elm", numReceivedMPs_host(0)); + Kokkos::View counter("counter", 1); + auto set_new_elem = PS_LAMBDA(const int& e, const int& mp, const bool& mask) { + if(mask){ + if (mpAppID(mp) == -1){ + auto xx=Kokkos::atomic_fetch_add(&counter(0), 1); + receivedMPs2Elm(xx) = e; + } + } + }; + parallel_for(set_new_elem, "countReceivedPtcls"); + //NEED AN ASSERT ELEMNT DOING COUNTER=#RECEIVED MPs + auto receivedMPs2Elm_host = Kokkos::create_mirror_view(receivedMPs2Elm); + Kokkos::deep_copy(receivedMPs2Elm_host, receivedMPs2Elm); + for(int i=0; i IntFunc; +typedef std::function IntIntFunc; + + enum MaterialPointSlice { MPF_Status = 0, @@ -124,7 +127,8 @@ class MaterialPoints { bool isRotatedFlag = false; Operating_Mode operating_mode; RebuildHelper rebuildFields; - IntFunc getAppID; + //IntFunc getAppID; + IntIntFunc getAppID; MPI_Comm mpi_comm; public: @@ -151,8 +155,10 @@ class MaterialPoints { typename std::enable_if::type setRebuildMPSlice(mpSliceData mpSliceIn); - void setAppIDFunc(IntFunc getAppIDIn); - int getNextAppID(); + //void setAppIDFunc(IntFunc getAppIDIn); + void setAppIDFunc(IntIntFunc getAppIDIn); + + int getNextAppID(int iElm); void rebuild() { IntView tgtElm("tgtElm", MPs->capacity()); From 8e310cfd67a772ce07fcad5e3c1f3523e21de62f Mon Sep 17 00:00:00 2001 From: nathd2 Date: Sat, 12 Apr 2025 07:59:05 -0400 Subject: [PATCH 14/42] iParticleNew Issue --- src/pmpo_MPMesh.cpp | 2 +- src/pmpo_c.cpp | 6 +++--- src/pmpo_c.h | 2 +- src/pmpo_defines.h | 2 +- src/pmpo_fortran.f90 | 3 ++- src/pmpo_materialPoints.cpp | 2 +- 6 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index 0509043..cd3fd33 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -345,7 +345,7 @@ void MPMesh::push(){ static int count=0; std::cout<<"Push"<<" "<473) exit(-1); Kokkos::Timer timer; p_mesh->computeRotLatLonIncr(); diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index 6616f9e..81df542 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -213,11 +213,11 @@ void polympo_setAppIDFunc_f(MPMesh_ptr p_mpmesh, IntVoidFunc getNext, void* appI } void polympo_setMPASAppIDFunc_f(MPMesh_ptr p_mpmesh, VoidVoidFunc getMPASAppID, void* arg1, void*arg2, void*arg3, - const int arg4) { + const int arg4, const int arg5) { checkMPMeshValid(p_mpmesh); auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - std::function polyMPO_getMPASAppID = [getMPASAppID, arg1, arg2, arg3, arg4](int iCell) - { int iParticleNew; getMPASAppID(arg1, arg2, arg3, arg4, iCell, iParticleNew); return iParticleNew;}; + std::function polyMPO_getMPASAppID = [getMPASAppID, arg1, arg2, arg3, arg4, arg5](int iCell) + { int iParticleNew; getMPASAppID(arg1, arg2, arg3, arg4, iCell, iParticleNew, arg5); return iParticleNew;}; p_MPs->setAppIDFunc(polyMPO_getMPASAppID); } diff --git a/src/pmpo_c.h b/src/pmpo_c.h index be83a83..13f36a8 100644 --- a/src/pmpo_c.h +++ b/src/pmpo_c.h @@ -25,7 +25,7 @@ void polympo_finishRebuildMPs_f(MPMesh_ptr p_mpmesh); void polympo_setAppIDFunc_f(MPMesh_ptr p_mpmesh, IntVoidFunc getNext, void* appIDs); void polympo_setMPASAppIDFunc_f(MPMesh_ptr p_mpmesh, VoidVoidFunc getMPASAppID, void* arg1, void* arg2, void*arg3, - const int arg4); + const int arg4, const int arg5); void polympo_getMPCurElmID_f(MPMesh_ptr p_mpmesh, const int numMPs, int* elmIDs); void polympo_setMPLatLonRotatedFlag_f(MPMesh_ptr p_mpmesh, const int isRotateFlag); diff --git a/src/pmpo_defines.h b/src/pmpo_defines.h index 21a7d49..05b301b 100644 --- a/src/pmpo_defines.h +++ b/src/pmpo_defines.h @@ -8,7 +8,7 @@ typedef void* MPMesh_ptr; //Function that receives void* and returns an int typedef int (*IntVoidFunc)(void*); -typedef void (*VoidVoidFunc)(void*, void*, void*, const int, int, int); +typedef void (*VoidVoidFunc)(void*, void*, void*, const int, int, int, const int); using space_t = Kokkos::DefaultExecutionSpace::memory_space; diff --git a/src/pmpo_fortran.f90 b/src/pmpo_fortran.f90 index 646a07d..e5670f3 100644 --- a/src/pmpo_fortran.f90 +++ b/src/pmpo_fortran.f90 @@ -118,7 +118,7 @@ subroutine polympo_setAppIDFunc(mpMesh, getNext, appIDs) & !> @param appIDs(in) Pointer to opaque data application data structure (that may contain all available app IDs) !--------------------------------------------------------------------------- subroutine polympo_setMPASAppIDFunc(mpMesh, getMPASAppID, & - arg1, arg2, arg3, arg4) & + arg1, arg2, arg3, arg4, arg5) & bind(C, NAME='polympo_setMPASAppIDFunc_f') use :: iso_c_binding type(c_ptr), value :: mpMesh @@ -127,6 +127,7 @@ subroutine polympo_setMPASAppIDFunc(mpMesh, getMPASAppID, & type(c_ptr), value :: arg2 type(c_ptr), value :: arg3 integer(c_int), intent(in), value :: arg4 + integer(c_int), intent(in), value :: arg5 end subroutine diff --git a/src/pmpo_materialPoints.cpp b/src/pmpo_materialPoints.cpp index 9bea799..2a90ba1 100644 --- a/src/pmpo_materialPoints.cpp +++ b/src/pmpo_materialPoints.cpp @@ -195,7 +195,7 @@ void MaterialPoints::migrate() { for(int i=0; i Date: Wed, 16 Apr 2025 12:56:15 -0400 Subject: [PATCH 15/42] New particleID can be found from MPAS in polyMPO --- src/pmpo_MPMesh.cpp | 22 +++++++++++++--- src/pmpo_c.cpp | 9 ++++--- src/pmpo_c.h | 3 +-- src/pmpo_defines.h | 2 +- src/pmpo_fortran.f90 | 8 +++--- src/pmpo_materialPoints.cpp | 50 ++++++++++++++++++++++++++----------- 6 files changed, 64 insertions(+), 30 deletions(-) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index cd3fd33..57afa6b 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -2,7 +2,7 @@ #include "pmpo_utils.hpp" #include "pmpo_MPMesh.hpp" #include "pmpo_wachspressBasis.hpp" - +#include namespace polyMPO{ void printVTP_mesh(MPMesh& mpMesh, int printVTPIndex=-1); @@ -345,7 +345,7 @@ void MPMesh::push(){ static int count=0; std::cout<<"Push"<<" "<473) exit(-1); + Kokkos::Timer timer; p_mesh->computeRotLatLonIncr(); @@ -358,7 +358,7 @@ void MPMesh::push(){ bool anyIsMigrating = false; do { - CVTTrackingElmCenterBased(count); // move to Tgt_XYZ + CVTTrackingElmCenterBased(); // move to Tgt_XYZ assert(cudaDeviceSynchronize() == cudaSuccess); p_MPs->updateMPSlice(); // Tgt_XYZ becomes Cur_XYZ p_MPs->updateMPSlice(); // Tgt becomes Cur @@ -369,12 +369,26 @@ void MPMesh::push(){ p_MPs->migrate(); else p_MPs->rebuild(); + printf("Done till here 0\n"); p_MPs->updateMPElmID(); //update mpElm IDs slices reconstructSlices(); } while (anyIsMigrating); - count ++; + + count ++; + + volatile int i = 0; + char hostname[256]; + gethostname(hostname, sizeof(hostname)); + MPI_Comm comm = p_MPs->getMPIComm(); + int comm_rank; + MPI_Comm_rank(comm, &comm_rank); + printf("Rank %d PID %d on %s ready for attach\n", comm_rank, getpid(), hostname); + fflush(stdout); + if(count==480) sleep(100); + + printf("Done till here\n"); pumipic::RecordTime("PolyMPO_push", timer.seconds()); } diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index 81df542..94e1589 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -212,12 +212,13 @@ void polympo_setAppIDFunc_f(MPMesh_ptr p_mpmesh, IntVoidFunc getNext, void* appI //p_MPs->setAppIDFunc(getNextAppID); } -void polympo_setMPASAppIDFunc_f(MPMesh_ptr p_mpmesh, VoidVoidFunc getMPASAppID, void* arg1, void*arg2, void*arg3, - const int arg4, const int arg5) { +//arg1 is blockPtr, arg2 is blockSize and arg3 is nCells +void polympo_setMPASAppIDFunc_f(MPMesh_ptr p_mpmesh, VoidVoidFunc getMPASAppID, void*arg1, + const int arg2, const int arg3) { checkMPMeshValid(p_mpmesh); auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - std::function polyMPO_getMPASAppID = [getMPASAppID, arg1, arg2, arg3, arg4, arg5](int iCell) - { int iParticleNew; getMPASAppID(arg1, arg2, arg3, arg4, iCell, iParticleNew, arg5); return iParticleNew;}; + std::function polyMPO_getMPASAppID = [getMPASAppID, arg1, arg2, arg3](int iCell) + { int iParticleNew=-1; getMPASAppID(arg1, arg2, arg3, iCell, iParticleNew); return iParticleNew;}; p_MPs->setAppIDFunc(polyMPO_getMPASAppID); } diff --git a/src/pmpo_c.h b/src/pmpo_c.h index 13f36a8..6dcbf4e 100644 --- a/src/pmpo_c.h +++ b/src/pmpo_c.h @@ -24,8 +24,7 @@ void polympo_startRebuildMPs_f(MPMesh_ptr p_mpmesh, const int numMPs, const int* void polympo_finishRebuildMPs_f(MPMesh_ptr p_mpmesh); void polympo_setAppIDFunc_f(MPMesh_ptr p_mpmesh, IntVoidFunc getNext, void* appIDs); -void polympo_setMPASAppIDFunc_f(MPMesh_ptr p_mpmesh, VoidVoidFunc getMPASAppID, void* arg1, void* arg2, void*arg3, - const int arg4, const int arg5); +void polympo_setMPASAppIDFunc_f(MPMesh_ptr p_mpmesh, VoidVoidFunc getMPASAppID, void* arg1, const int arg2, const int arg3); void polympo_getMPCurElmID_f(MPMesh_ptr p_mpmesh, const int numMPs, int* elmIDs); void polympo_setMPLatLonRotatedFlag_f(MPMesh_ptr p_mpmesh, const int isRotateFlag); diff --git a/src/pmpo_defines.h b/src/pmpo_defines.h index 05b301b..14435eb 100644 --- a/src/pmpo_defines.h +++ b/src/pmpo_defines.h @@ -8,7 +8,7 @@ typedef void* MPMesh_ptr; //Function that receives void* and returns an int typedef int (*IntVoidFunc)(void*); -typedef void (*VoidVoidFunc)(void*, void*, void*, const int, int, int, const int); +typedef void (*VoidVoidFunc)(void*, const int, const int, const int, int&); using space_t = Kokkos::DefaultExecutionSpace::memory_space; diff --git a/src/pmpo_fortran.f90 b/src/pmpo_fortran.f90 index e5670f3..c0cd8ca 100644 --- a/src/pmpo_fortran.f90 +++ b/src/pmpo_fortran.f90 @@ -118,16 +118,14 @@ subroutine polympo_setAppIDFunc(mpMesh, getNext, appIDs) & !> @param appIDs(in) Pointer to opaque data application data structure (that may contain all available app IDs) !--------------------------------------------------------------------------- subroutine polympo_setMPASAppIDFunc(mpMesh, getMPASAppID, & - arg1, arg2, arg3, arg4, arg5) & + arg1, arg2, arg3) & bind(C, NAME='polympo_setMPASAppIDFunc_f') use :: iso_c_binding type(c_ptr), value :: mpMesh type(c_funptr), value :: getMPASAppID type(c_ptr), value :: arg1 - type(c_ptr), value :: arg2 - type(c_ptr), value :: arg3 - integer(c_int), intent(in), value :: arg4 - integer(c_int), intent(in), value :: arg5 + integer(c_int), intent(in), value :: arg2 + integer(c_int), intent(in), value :: arg3 end subroutine diff --git a/src/pmpo_materialPoints.cpp b/src/pmpo_materialPoints.cpp index 2a90ba1..0aa7f40 100644 --- a/src/pmpo_materialPoints.cpp +++ b/src/pmpo_materialPoints.cpp @@ -158,9 +158,9 @@ void MaterialPoints::migrate() { parallel_for(setMigrationFields, "setMigrationFields"); MPs->migrate(new_elem, new_process); - //SINCE REBUILT, mpAppID needs to be be recalled + //AS REBUILT, mpAppID needs to be be recalled mpAppID = getData(); - //Count MPs that have -1 appID, so that we can count no of MPs received + //Count MPs that have -1 appID, so that we can count no of MPs received by a rank Kokkos::View numReceivedMPs("numReceivedMPs", 1); Kokkos::deep_copy(numReceivedMPs, 0); auto countnewMPs = PS_LAMBDA(const int& e, const int& mp, const bool& mask) { @@ -170,34 +170,56 @@ void MaterialPoints::migrate() { } }; parallel_for(countnewMPs, "countReceivedPtcls"); - Kokkos::fence(); - //Bring them to CPU and print particles received auto numReceivedMPs_host = Kokkos::create_mirror_view(numReceivedMPs); Kokkos::deep_copy(numReceivedMPs_host, numReceivedMPs); - if(numReceivedMPs_host(0)) - std::cout <<"Rank "< receivedMPs2Elm("ReceivedMPs2Elm", numReceivedMPs_host(0)); Kokkos::View counter("counter", 1); + Kokkos::deep_copy(counter, 0); auto set_new_elem = PS_LAMBDA(const int& e, const int& mp, const bool& mask) { if(mask){ if (mpAppID(mp) == -1){ - auto xx=Kokkos::atomic_fetch_add(&counter(0), 1); - receivedMPs2Elm(xx) = e; + auto count_temp=Kokkos::atomic_fetch_add(&counter(0), 1); + receivedMPs2Elm(count_temp) = e; } } }; - parallel_for(set_new_elem, "countReceivedPtcls"); - //NEED AN ASSERT ELEMNT DOING COUNTER=#RECEIVED MPs + parallel_for(set_new_elem, "countReceivedPtcls"); + //Bring them to CPU so that elm_id can be passed and a new mpAppID can be found from CPU auto receivedMPs2Elm_host = Kokkos::create_mirror_view(receivedMPs2Elm); Kokkos::deep_copy(receivedMPs2Elm_host, receivedMPs2Elm); - + auto counter_host = Kokkos::create_mirror_view(counter); + Kokkos::deep_copy(counter_host, counter); + assert(numReceivedMPs_host(0)==counter_host(0)); + if(counter_host(0)) + std::cout <<"Rank "< appIDs; for(int i=0; i appIDs_host(appIDs.data(), appIDs.size()); + Kokkos::View appIDs_d("appIDsDevice", appIDs.size()); + Kokkos::deep_copy(appIDs_d, appIDs_host); + //If the mpAppID is -1 assign a new mpAppID + Kokkos::deep_copy(counter, 0); + auto set_appID = PS_LAMBDA(const int& e, const int& mp, const bool& mask) { + if(mask){ + if (mpAppID(mp) == -1){ + auto count_temp=Kokkos::atomic_fetch_add(&counter(0), 1); + mpAppID(mp)=appIDs_d(count_temp)-1; + printf("Count_temp %d and new appID %d \n", count_temp, mpAppID(mp)); + } + } + }; + parallel_for(set_appID, "setApplicationIDs"); + if (getOpMode() == polyMPO::MP_DEBUG) printf("Material point migration: %f\n", timer.seconds()); pumipic::RecordTime("PolyMPO_migrate", timer.seconds()); From a8582e86e14493c0fe052918b64025e8cd169d86 Mon Sep 17 00:00:00 2001 From: nathd2 Date: Thu, 24 Apr 2025 09:37:22 -0400 Subject: [PATCH 16/42] Before Phase 1 --- src/pmpo_MPMesh.cpp | 11 +++++------ src/pmpo_materialPoints.cpp | 4 +--- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index 57afa6b..06b13d8 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -369,7 +369,6 @@ void MPMesh::push(){ p_MPs->migrate(); else p_MPs->rebuild(); - printf("Done till here 0\n"); p_MPs->updateMPElmID(); //update mpElm IDs slices reconstructSlices(); @@ -384,11 +383,11 @@ void MPMesh::push(){ MPI_Comm comm = p_MPs->getMPIComm(); int comm_rank; MPI_Comm_rank(comm, &comm_rank); - printf("Rank %d PID %d on %s ready for attach\n", comm_rank, getpid(), hostname); - fflush(stdout); - if(count==480) sleep(100); - - printf("Done till here\n"); + if(count==480){ + printf("Rank %d PID %d on %s ready for attach\n", comm_rank, getpid(), hostname); + fflush(stdout); + sleep(100); + } pumipic::RecordTime("PolyMPO_push", timer.seconds()); } diff --git a/src/pmpo_materialPoints.cpp b/src/pmpo_materialPoints.cpp index 0aa7f40..76218aa 100644 --- a/src/pmpo_materialPoints.cpp +++ b/src/pmpo_materialPoints.cpp @@ -151,7 +151,6 @@ void MaterialPoints::migrate() { new_process(mp) = MPs2Proc(mp); if(rank!=new_process(mp)){ mpAppID(mp)=-1; - printf("Particle migrated and so its AppID is -1\n"); } } }; @@ -200,7 +199,7 @@ void MaterialPoints::migrate() { std::vector appIDs; for(int i=0; i appIDs_host(appIDs.data(), appIDs.size()); @@ -214,7 +213,6 @@ void MaterialPoints::migrate() { if (mpAppID(mp) == -1){ auto count_temp=Kokkos::atomic_fetch_add(&counter(0), 1); mpAppID(mp)=appIDs_d(count_temp)-1; - printf("Count_temp %d and new appID %d \n", count_temp, mpAppID(mp)); } } }; From 3988a6aee621216a30668deb225854443ca556b1 Mon Sep 17 00:00:00 2001 From: nathd2 Date: Wed, 30 Apr 2025 08:58:42 -0400 Subject: [PATCH 17/42] Push operation from MPAS in while loop --- src/pmpo_MPMesh.cpp | 35 ++++++++++++++++++++-- src/pmpo_MPMesh.hpp | 1 + src/pmpo_c.cpp | 59 +++++++++++++++++++++++++++++++++++-- src/pmpo_c.h | 7 ++++- src/pmpo_fortran.f90 | 35 +++++++++++++++++++++- src/pmpo_materialPoints.cpp | 19 +++++++++--- src/pmpo_materialPoints.hpp | 7 ++++- 7 files changed, 151 insertions(+), 12 deletions(-) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index 06b13d8..f15f2ff 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -141,9 +141,9 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ printVTP_mesh(comm_rank); } - assert(cudaDeviceSynchronize()==cudaSuccess); - MPI_Barrier(MPI_COMM_WORLD); - + //assert(cudaDeviceSynchronize()==cudaSuccess); + //MPI_Barrier(MPI_COMM_WORLD); + printf("NumMPs %d \n", numMPs); Vec3dView history("positionHistory",numMPs); Vec3dView resultLeft("positionResult",numMPs); Vec3dView resultRight("positionResult",numMPs); @@ -341,6 +341,35 @@ bool getAnyIsMigrating(MaterialPoints* p_MPs, bool isMigrating) { return anyIsMigrating; } +bool MPMesh::push1P(){ + + static int count=0; + std::cout<<"Push1P"<<" "<computeRotLatLonIncr(); + sphericalInterpolation(*this); + + //Push the MPs + p_MPs->updateRotLatLonAndXYZ2Tgt(p_mesh->getSphereRadius()); + + //Given target location find the new MP element and the process it belongs to + CVTTrackingElmCenterBased(); + //From the above two inputs find if any particle needs to be migrated + bool anyIsMigrating = getAnyIsMigrating(p_MPs, p_MPs->check_migrate()); + + //New element (maynot be final) so that MPAS can do the migration + p_MPs->updateMPElmID(); + + + p_MPs->updateMPSlice(); + p_MPs->updateMPSlice(); + + count++; + return anyIsMigrating; + +} + void MPMesh::push(){ static int count=0; diff --git a/src/pmpo_MPMesh.hpp b/src/pmpo_MPMesh.hpp index 33f0599..b30bcf3 100644 --- a/src/pmpo_MPMesh.hpp +++ b/src/pmpo_MPMesh.hpp @@ -43,6 +43,7 @@ class MPMesh{ void CVTTrackingEdgeCenterBased(Vec2dView dx); void CVTTrackingElmCenterBased(const int printVTPIndex = -1); void T2LTracking(Vec2dView dx); + bool push1P(); void push(); void calcBasis(); diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index 94e1589..85046a8 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -110,6 +110,7 @@ void polympo_createMPs_f(MPMesh_ptr p_mpmesh, fprintf(stderr,"The minElmID is incorrect! Offset is wrong!\n"); exit(1); } + printf("Offset %d \n", offset); std::vector active_mpIDs(numMPs); std::vector active_mp2Elm(numMPs); @@ -198,6 +199,43 @@ void polympo_startRebuildMPs_f(MPMesh_ptr p_mpmesh, } } +void polympo_startRebuildMPs_f2(MPMesh_ptr p_mpmesh, + const int sizeMP2elm, + const int* elem_ids, + const int nMPs_delete, + const int nMPs_add, + const int* recvMPs_elm, + const int* recvMPs_ids) { + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + int offset = p_MPs->getElmIDoffset(); + printf("To delete %d MPs add %d MPs\n", nMPs_delete, nMPs_add); + auto elem_ids_d = create_mirror_view_and_copy(elem_ids, sizeMP2elm); + auto recvMPs_elm_d = create_mirror_view_and_copy(recvMPs_elm, nMPs_add); + auto recvMPs_ids_d = create_mirror_view_and_copy(recvMPs_ids, nMPs_add); + + Kokkos::View mp2Elm("mp2Elm", p_MPs->getCapacity()); + Kokkos::View numDeletedMPs_d("numDeletedMPs", 1); + auto mpAppID = p_MPs->getData(); + + auto setMP2Elm = PS_LAMBDA(const int& e, const int& mp, const int& mask) { + if(mask) { + mp2Elm(mp) = elem_ids_d(mpAppID(mp)); + if (mp2Elm(mp) == MP_DELETE){ + Kokkos::atomic_increment(&numDeletedMPs_d(0)); + printf("Deleted particle is %d it was in %d \n", mpAppID(mp), e); + } + } + }; + p_MPs->parallel_for(setMP2Elm, "setMP2Elm"); + int numDeletedMPs = pumipic::getLastValue(numDeletedMPs_d); + if(numDeletedMPs) printf("Dleted %d particles \n", numDeletedMPs); + + p_MPs->startRebuild(mp2Elm, nMPs_add, recvMPs_elm_d, recvMPs_ids_d); +} + + + void polympo_finishRebuildMPs_f(MPMesh_ptr p_mpmesh) { checkMPMeshValid(p_mpmesh); @@ -255,13 +293,14 @@ void polympo_setMPLatLonRotatedFlag_f(MPMesh_ptr p_mpmesh, const int isRotateFla void polympo_setMPPositions_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, - const double* mpPositionsIn){ + const double* mpPositionsIn, + const int setTgtPos){ Kokkos::Timer timer; checkMPMeshValid(p_mpmesh); auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; PMT_ALWAYS_ASSERT(nComps == vec3d_nEntries); PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); kkViewHostU mpPositionsIn_h(mpPositionsIn,nComps,numMPs); if (p_MPs->rebuildOngoing()) { @@ -270,6 +309,10 @@ void polympo_setMPPositions_f(MPMesh_ptr p_mpmesh, } auto mpPositions = p_MPs->getData(); + if(setTgtPos){ + printf("Setting the TGT position instead\n"); + mpPositions = p_MPs->getData(); + } auto mpAppID = p_MPs->getData(); Kokkos::View mpPositionsIn_d("mpPositionsDevice",vec3d_nEntries,numMPs); Kokkos::deep_copy(mpPositionsIn_d, mpPositionsIn_h); @@ -281,6 +324,7 @@ void polympo_setMPPositions_f(MPMesh_ptr p_mpmesh, } }; p_MPs->parallel_for(setPos, "setMPPositions"); + printf("Done setting MP pos\n"); pumipic::RecordTime("PolyMPO_setMPPositions", timer.seconds()); } @@ -987,6 +1031,12 @@ void polympo_getMeshVtxOnSurfDispIncr_f(MPMesh_ptr p_mpmesh, const int nComps, c Kokkos::deep_copy(arrayHost, array_d); } +bool polympo_push1P_f(MPMesh_ptr p_mpmesh){ + checkMPMeshValid(p_mpmesh); + bool is_migrating=((polyMPO::MPMesh*)p_mpmesh) ->push1P(); + return is_migrating; +} + void polympo_push_f(MPMesh_ptr p_mpmesh){ checkMPMeshValid(p_mpmesh); ((polyMPO::MPMesh*)p_mpmesh) ->push(); @@ -1068,6 +1118,11 @@ void polympo_setElmGlobal_f(MPMesh_ptr p_mpmesh, const int nCells, const int* ar } +int polympo_getMPCount_f(MPMesh_ptr p_mpmesh) { + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + return p_MPs->getCount(); // This returns the int you're after +} + void polympo_enableTiming_f(){ pumipic::EnableTiming(); } diff --git a/src/pmpo_c.h b/src/pmpo_c.h index 6dcbf4e..3ab2331 100644 --- a/src/pmpo_c.h +++ b/src/pmpo_c.h @@ -21,6 +21,10 @@ void polympo_setMPICommunicator_f(MPMesh_ptr p_mpmesh, MPI_Fint fcomm); //MP info void polympo_createMPs_f(MPMesh_ptr p_mpmesh, const int numElms, const int numMPs, int* mpsPerElm, const int* mp2Elm, const int* isMPActive); void polympo_startRebuildMPs_f(MPMesh_ptr p_mpmesh, const int numMPs, const int* allTgtMpElmIn, const int* addedMPMask); +void polympo_startRebuildMPs_f2(MPMesh_ptr p_mpmesh, const int size1, const int* arg1, const int size2, const int size3, const int* arg2, const int* arg3); + +int polympo_getMPCount_f(MPMesh_ptr p_mpmesh); + void polympo_finishRebuildMPs_f(MPMesh_ptr p_mpmesh); void polympo_setAppIDFunc_f(MPMesh_ptr p_mpmesh, IntVoidFunc getNext, void* appIDs); @@ -30,7 +34,7 @@ void polympo_getMPCurElmID_f(MPMesh_ptr p_mpmesh, const int numMPs, int* elmIDs) void polympo_setMPLatLonRotatedFlag_f(MPMesh_ptr p_mpmesh, const int isRotateFlag); //MP slices -void polympo_setMPPositions_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpPositionsIn); +void polympo_setMPPositions_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpPositionsIn, const int setTgtPos); void polympo_getMPPositions_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, double* mpPositionsIn); void polympo_setMPRotLatLon_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpRotLatLonIn); void polympo_getMPRotLatLon_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, double* mpRotLatLonHost); @@ -90,6 +94,7 @@ void polympo_getMeshDualTriangleArea_f(MPMesh_ptr p_mpmesh, const int nVertices, // calculations void polympo_push_f(MPMesh_ptr p_mpmesh); +bool polympo_push1P_f(MPMesh_ptr p_mpmesh); // Reconstruction of variables from MPs to mesh vertices void polympo_setReconstructionOfMass_f(MPMesh_ptr p_mpmesh, const int order, const int meshEntType); diff --git a/src/pmpo_fortran.f90 b/src/pmpo_fortran.f90 index c0cd8ca..4efe641 100644 --- a/src/pmpo_fortran.f90 +++ b/src/pmpo_fortran.f90 @@ -87,6 +87,20 @@ subroutine polympo_startRebuildMPs(mpMesh, numMPs, allMP2Elm, addedMPMask) & type(c_ptr), intent(in), value :: allMP2Elm type(c_ptr), intent(in), value :: addedMPMask end subroutine + + + subroutine polympo_startRebuildMPs2(mpMesh, size1, arg1, size2, size3, arg2, arg3) & + bind(C, NAME='polympo_startRebuildMPs_f2') + use :: iso_c_binding + type(c_ptr), value :: mpMesh + integer(c_int), value :: size1 + type(c_ptr), intent(in), value :: arg1 + integer(c_int), value :: size2 + integer(c_int), value :: size3 + type(c_ptr), intent(in), value :: arg2 + type(c_ptr), intent(in), value :: arg3 + end subroutine + ! !--------------------------------------------------------------------------- !> @brief called after startRebuild() !> @brief called after initializing MP fields @@ -160,12 +174,13 @@ subroutine polympo_setMPLatLonRotatedFlag(mpMesh, isRotateFlag) & !> @param numMPs(in) number of the MPs !> @param array(in) MP current position 2D array (3,numMPs), allocated by user on host !--------------------------------------------------------------------------- - subroutine polympo_setMPPositions(mpMesh, nComps, numMPs, array) & + subroutine polympo_setMPPositions(mpMesh, nComps, numMPs, array, setTgtPos) & bind(C, NAME='polympo_setMPPositions_f') use :: iso_c_binding type(c_ptr), value :: mpMesh integer(c_int), value :: nComps, numMPs type(c_ptr), value :: array + integer(c_int), value:: setTgtPos end subroutine !--------------------------------------------------------------------------- !> @brief get the MP positions array from a polympo array @@ -743,6 +758,16 @@ subroutine polympo_push(mpMesh) & use :: iso_c_binding type(c_ptr), value :: mpMesh end subroutine + + + !--------------------------------------------------------------------------- + !> @brief calculate the MPs from given mesh vertices rotational latitude + logical function polympo_push1P(mpMesh) & + bind(C, NAME='polympo_push1P_f') + use :: iso_c_binding + type(c_ptr), value :: mpMesh + end function + !--------------------------------------------------------------------------- !> @brief start the reconstruction of MP Mass to Mesh Vertices !> @param mpmesh(in/out) MPMesh object @@ -823,6 +848,13 @@ subroutine polympo_setTimingVerbosity(v) bind(C, NAME='polympo_setTimingVerbosit use :: iso_c_binding integer(c_int), value :: v end subroutine + + integer function polympo_getMPCount(mpMesh) bind(C, name="polympo_getMPCount_f") + use :: iso_c_binding + implicit none + type(c_ptr), value :: mpMesh + end function + end interface contains !--------------------------------------------------------------------------- @@ -838,4 +870,5 @@ subroutine polympo_checkPrecisionForRealKind(APP_RKIND) call exit(1) end if end subroutine + end module diff --git a/src/pmpo_materialPoints.cpp b/src/pmpo_materialPoints.cpp index 76218aa..aca7f2e 100644 --- a/src/pmpo_materialPoints.cpp +++ b/src/pmpo_materialPoints.cpp @@ -95,6 +95,16 @@ void MaterialPoints::startRebuild(IntView tgtElm, int addedNumMPs, IntView added rebuildFields.addedSlices_h = createInternalMemberViews(addedNumMPs, addedMP2elm_h, addedMPAppID_h); } +void MaterialPoints::startRebuild(IntView tgtElm, int addedNumMPs, IntView addedMP2elm, IntView addedMPAppID) { + rebuildFields.ongoing = true; + rebuildFields.addedNumMPs = addedNumMPs; + rebuildFields.addedMP2elm = addedMP2elm; + rebuildFields.allTgtElm = tgtElm; + auto addedMP2elm_h = Kokkos::create_mirror_view_and_copy(hostSpace(), addedMP2elm); + auto addedMPAppID_h = Kokkos::create_mirror_view_and_copy(hostSpace(), addedMPAppID); + rebuildFields.addedSlices_h = createInternalMemberViews(addedNumMPs, addedMP2elm_h, addedMPAppID_h); +} + void MaterialPoints::finishRebuild() { auto addedSlices_d = ps::createMemberViews(rebuildFields.addedNumMPs); ps::CopyMemSpaceToMemSpace(addedSlices_d, rebuildFields.addedSlices_h); @@ -114,20 +124,20 @@ void MaterialPoints::setMPIComm(MPI_Comm comm) { } bool MaterialPoints::check_migrate(){ - Kokkos::Timer timer; - auto MPs2Elm = getData(); + + Kokkos::Timer timer; auto MPs2Proc = getData(); - IntView isMigrating("isMigrating", 1); - int rank; MPI_Comm_rank(mpi_comm, &rank); + auto setMigrationFields = PS_LAMBDA(const int& e, const int& mp, const bool& mask) { if (mask) { if (MPs2Proc(mp) != rank) isMigrating(0) = 1; } }; parallel_for(setMigrationFields, "setMigrationFields"); + if (getOpMode() == polyMPO::MP_DEBUG) printf("Material point check migration: %f\n", timer.seconds()); pumipic::RecordTime("PolyMPO_check_migrate", timer.seconds()); @@ -150,6 +160,7 @@ void MaterialPoints::migrate() { new_elem(mp) = MPs2Elm(mp); new_process(mp) = MPs2Proc(mp); if(rank!=new_process(mp)){ + printf("Particle %d in rank %d to be moved from %d %d \n", mpAppID(mp), rank, e, new_elem(mp) ); mpAppID(mp)=-1; } } diff --git a/src/pmpo_materialPoints.hpp b/src/pmpo_materialPoints.hpp index 13a9a86..c2a037f 100644 --- a/src/pmpo_materialPoints.hpp +++ b/src/pmpo_materialPoints.hpp @@ -139,6 +139,8 @@ class MaterialPoints { void rebuild(IntView addedMP2elm, IntView addedMPAppID); void startRebuild(IntView tgtElm, int addedNumMPs, IntView addedMP2elm, IntView addedMPAppID, Kokkos::View addedMPMask); + void startRebuild(IntView tgtElm, int addedNumMPs, IntView addedMP2elm, IntView addedMPAppID); + void finishRebuild(); bool rebuildOngoing(); @@ -163,9 +165,12 @@ class MaterialPoints { void rebuild() { IntView tgtElm("tgtElm", MPs->capacity()); auto tgtMpElm = MPs->get(); - auto setTgtElm = PS_LAMBDA(const int&, const int& mp, const int& mask) { + auto mpAppID = MPs->get(); + auto setTgtElm = PS_LAMBDA(const int& e, const int& mp, const int& mask) { if(mask) { + auto app_id=mpAppID(mp); tgtElm(mp) = tgtMpElm(mp); + if(app_id==191) printf("Going from elm %d to %d \n", e, tgtMpElm(mp)); } }; ps::parallel_for(MPs, setTgtElm, "setTargetElement"); From 77cde5b1ae1d00be4e1198149ee4dc1e272f3d85 Mon Sep 17 00:00:00 2001 From: nathd2 Date: Mon, 5 May 2025 22:14:27 -0400 Subject: [PATCH 18/42] Woking 2 proc case --- src/pmpo_MPMesh.cpp | 54 ++++++---- src/pmpo_MPMesh.hpp | 3 + src/pmpo_c.cpp | 204 ++++++++++++++++++++++++++++++++---- src/pmpo_c.h | 15 ++- src/pmpo_fortran.f90 | 105 ++++++++++++++++++- src/pmpo_materialPoints.cpp | 18 ++++ 6 files changed, 355 insertions(+), 44 deletions(-) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index f15f2ff..f2fa0c6 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -128,7 +128,8 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ auto MPs2Proc = p_MPs->getData(); auto elm2Process = p_mesh->getElm2Process(); auto elm2global = p_mesh->getElmGlobal(); - + auto mpAppID = p_MPs->getData(); + MPI_Comm comm = p_MPs->getMPIComm(); int comm_rank; MPI_Comm_rank(comm, &comm_rank); @@ -143,12 +144,14 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ //assert(cudaDeviceSynchronize()==cudaSuccess); //MPI_Barrier(MPI_COMM_WORLD); - printf("NumMPs %d \n", numMPs); Vec3dView history("positionHistory",numMPs); Vec3dView resultLeft("positionResult",numMPs); Vec3dView resultRight("positionResult",numMPs); Vec3dView mpTgtPosArray("positionTarget",numMPs); Kokkos::View counter("counter",1); + + assert(cudaDeviceSynchronize() == cudaSuccess); + //printf("Rank %d Foo4 Begin\n", comm_rank); auto CVTElmCalc = PS_LAMBDA(const int& elm, const int& mp, const int&mask){ Vec3d MP(mpPositions(mp,0),mpPositions(mp,1),mpPositions(mp,2)); @@ -181,13 +184,15 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ if(closestElm<0){ MPs2Elm(mp) = iElm; - if (elm2Process.size() > 0) - MPs2Proc(mp) = elm2Process(iElm); + MPs2Proc(mp) = elm2Process(iElm); break; }else{ iElm = closestElm; } } + if(mpAppID(mp)==0 || mpAppID(mp)==191) + printf("Pos %.15e %.15e %.15e => %.15e %.15e %.15e\n", mpPositions(mp,0), mpPositions(mp,1), + mpPositions(mp,2), mpTgtPos(mp,0), mpTgtPos(mp,1), mpTgtPos(mp, 2)); if(printVTPIndex>=0 && numMPs>0){ //printf("Rank %d mp %d counter %d \n", comm_rank, mp, counter); @@ -211,6 +216,10 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ } }; p_MPs->parallel_for(CVTElmCalc,"CVTTrackingElmCenterBasedCalc"); + + assert(cudaDeviceSynchronize() == cudaSuccess); + //printf("Rank %d Foo4 End\n", comm_rank); + if(printVTPIndex>=0 && numMPs>0){ Vec3dView::HostMirror h_history = Kokkos::create_mirror_view(history); @@ -341,35 +350,44 @@ bool getAnyIsMigrating(MaterialPoints* p_MPs, bool isMigrating) { return anyIsMigrating; } -bool MPMesh::push1P(){ - - static int count=0; - std::cout<<"Push1P"<<" "<computeRotLatLonIncr(); sphericalInterpolation(*this); - //Push the MPs p_MPs->updateRotLatLonAndXYZ2Tgt(p_mesh->getSphereRadius()); - - //Given target location find the new MP element and the process it belongs to + count0 ++; +} + +bool MPMesh::push1P(){ + //Given target location find the new element or the last element in a partioned mesh + //and the process it belongs to so that migration can be checked + static int count_p=0; CVTTrackingElmCenterBased(); + //From the above two inputs find if any particle needs to be migrated bool anyIsMigrating = getAnyIsMigrating(p_MPs, p_MPs->check_migrate()); + count_p=count_p+1; + return anyIsMigrating; - //New element (maynot be final) so that MPAS can do the migration +} + +void MPMesh::push_swap(){ + //current becomes target, target becomes -1 p_MPs->updateMPElmID(); - +} + +void MPMesh::push_swap_pos(){ + //current becomes target, target becomes -1 + //Making read for next push_ahead p_MPs->updateMPSlice(); p_MPs->updateMPSlice(); - - count++; - return anyIsMigrating; - } + void MPMesh::push(){ static int count=0; diff --git a/src/pmpo_MPMesh.hpp b/src/pmpo_MPMesh.hpp index b30bcf3..7b54afb 100644 --- a/src/pmpo_MPMesh.hpp +++ b/src/pmpo_MPMesh.hpp @@ -44,6 +44,9 @@ class MPMesh{ void CVTTrackingElmCenterBased(const int printVTPIndex = -1); void T2LTracking(Vec2dView dx); bool push1P(); + void push_ahead(); + void push_swap(); + void push_swap_pos(); void push(); void calcBasis(); diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index 85046a8..ce7b7ba 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -204,12 +204,24 @@ void polympo_startRebuildMPs_f2(MPMesh_ptr p_mpmesh, const int* elem_ids, const int nMPs_delete, const int nMPs_add, - const int* recvMPs_elm, - const int* recvMPs_ids) { + int* recvMPs_elm, + int* recvMPs_ids) { checkMPMeshValid(p_mpmesh); auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; int offset = p_MPs->getElmIDoffset(); - printf("To delete %d MPs add %d MPs\n", nMPs_delete, nMPs_add); + + int rank; + auto mpi_comm=p_MPs->getMPIComm(); + MPI_Comm_rank(mpi_comm, &rank); + + printf("Rank %d to add %d and delete %d ptcls \n", rank, nMPs_add, nMPs_delete); + for (int k=0; k mp2Elm("mp2Elm", p_MPs->getCapacity()); Kokkos::View numDeletedMPs_d("numDeletedMPs", 1); auto mpAppID = p_MPs->getData(); - auto setMP2Elm = PS_LAMBDA(const int& e, const int& mp, const int& mask) { if(mask) { - mp2Elm(mp) = elem_ids_d(mpAppID(mp)); - if (mp2Elm(mp) == MP_DELETE){ + mp2Elm(mp) = elem_ids_d(mpAppID(mp))-offset; + if (mp2Elm(mp) == MP_DELETE){ + //printf("Rank %d to delete %d particle in element %d \n", rank, mpAppID(mp), e); Kokkos::atomic_increment(&numDeletedMPs_d(0)); - printf("Deleted particle is %d it was in %d \n", mpAppID(mp), e); - } + } } }; p_MPs->parallel_for(setMP2Elm, "setMP2Elm"); int numDeletedMPs = pumipic::getLastValue(numDeletedMPs_d); - if(numDeletedMPs) printf("Dleted %d particles \n", numDeletedMPs); - + assert(nMPs_delete==numDeletedMPs); p_MPs->startRebuild(mp2Elm, nMPs_add, recvMPs_elm_d, recvMPs_ids_d); } @@ -260,13 +270,42 @@ void polympo_setMPASAppIDFunc_f(MPMesh_ptr p_mpmesh, VoidVoidFunc getMPASAppID, p_MPs->setAppIDFunc(polyMPO_getMPASAppID); } +void polympo_getMPTgtElmID_f(MPMesh_ptr p_mpmesh, + const int numMPs, + int* elmIDs){ + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); + //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + auto mpTgtElmID = p_MPs->getData(); + auto mpAppID = p_MPs->getData(); + auto elmIDoffset = p_MPs->getElmIDoffset(); + + kkIntViewHostU arrayHost(elmIDs,numMPs); + polyMPO::IntView mpTgtElmIDCopy("mpTgtElmIDNewValue",numMPs); + + int rank; + auto mpi_comm=p_MPs->getMPIComm(); + MPI_Comm_rank(mpi_comm, &rank); + + auto setTgtElmId = PS_LAMBDA(const int&, const int& mp, const int& mask){ + if(mask){ + mpTgtElmIDCopy(mpAppID(mp)) = mpTgtElmID(mp)+elmIDoffset; + } + }; + p_MPs->parallel_for(setTgtElmId, "set mpTgtElmID"); + Kokkos::deep_copy( arrayHost, mpTgtElmIDCopy); + +} + void polympo_getMPCurElmID_f(MPMesh_ptr p_mpmesh, const int numMPs, int* elmIDs){ + checkMPMeshValid(p_mpmesh); auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); auto mpCurElmID = p_MPs->getData(); auto mpAppID = p_MPs->getData(); auto elmIDoffset = p_MPs->getElmIDoffset(); @@ -274,13 +313,18 @@ void polympo_getMPCurElmID_f(MPMesh_ptr p_mpmesh, kkIntViewHostU arrayHost(elmIDs,numMPs); polyMPO::IntView mpCurElmIDCopy("mpCurElmIDNewValue",numMPs); + int rank; + auto mpi_comm=p_MPs->getMPIComm(); + MPI_Comm_rank(mpi_comm, &rank); + auto getElmId = PS_LAMBDA(const int&, const int& mp, const int& mask){ if(mask){ - mpCurElmIDCopy(mpAppID(mp)) = mpCurElmID(mp)+elmIDoffset; + mpCurElmIDCopy(mpAppID(mp)) = mpCurElmID(mp)+elmIDoffset; } }; p_MPs->parallel_for(getElmId, "get mpCurElmID"); Kokkos::deep_copy( arrayHost, mpCurElmIDCopy); + } void polympo_setMPLatLonRotatedFlag_f(MPMesh_ptr p_mpmesh, const int isRotateFlag){ @@ -293,8 +337,7 @@ void polympo_setMPLatLonRotatedFlag_f(MPMesh_ptr p_mpmesh, const int isRotateFla void polympo_setMPPositions_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, - const double* mpPositionsIn, - const int setTgtPos){ + const double* mpPositionsIn){ Kokkos::Timer timer; checkMPMeshValid(p_mpmesh); auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; @@ -309,11 +352,8 @@ void polympo_setMPPositions_f(MPMesh_ptr p_mpmesh, } auto mpPositions = p_MPs->getData(); - if(setTgtPos){ - printf("Setting the TGT position instead\n"); - mpPositions = p_MPs->getData(); - } auto mpAppID = p_MPs->getData(); + Kokkos::View mpPositionsIn_d("mpPositionsDevice",vec3d_nEntries,numMPs); Kokkos::deep_copy(mpPositionsIn_d, mpPositionsIn_h); auto setPos = PS_LAMBDA(const int&, const int& mp, const int& mask){ @@ -324,7 +364,6 @@ void polympo_setMPPositions_f(MPMesh_ptr p_mpmesh, } }; p_MPs->parallel_for(setPos, "setMPPositions"); - printf("Done setting MP pos\n"); pumipic::RecordTime("PolyMPO_setMPPositions", timer.seconds()); } @@ -336,7 +375,7 @@ void polympo_getMPPositions_f(MPMesh_ptr p_mpmesh, auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; PMT_ALWAYS_ASSERT(nComps == vec3d_nEntries); PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); auto mpPositions = p_MPs->getData(); auto mpAppID = p_MPs->getData(); @@ -353,6 +392,66 @@ void polympo_getMPPositions_f(MPMesh_ptr p_mpmesh, Kokkos::deep_copy(arrayHost, mpPositionsCopy); } + +void polympo_setMPTgtPositions_f(MPMesh_ptr p_mpmesh, + const int nComps, + const int numMPs, + const double* mpPositionsIn){ + Kokkos::Timer timer; + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + PMT_ALWAYS_ASSERT(nComps == vec3d_nEntries); + PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); + //printf("NumMPs %d %d \n", numMPs, p_MPs->getCount()); + //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + kkViewHostU mpPositionsIn_h(mpPositionsIn,nComps,numMPs); + + if (p_MPs->rebuildOngoing()) { + p_MPs->setRebuildMPSlice(mpPositionsIn_h); + return; + } + + auto mpPositions = p_MPs->getData(); + auto mpAppID = p_MPs->getData(); + Kokkos::View mpPositionsIn_d("mpPositionsDevice",vec3d_nEntries,numMPs); + Kokkos::deep_copy(mpPositionsIn_d, mpPositionsIn_h); + auto setPos = PS_LAMBDA(const int&, const int& mp, const int& mask){ + if(mask){ + mpPositions(mp,0) = mpPositionsIn_d(0, mpAppID(mp)); + mpPositions(mp,1) = mpPositionsIn_d(1, mpAppID(mp)); + mpPositions(mp,2) = mpPositionsIn_d(2, mpAppID(mp)); + } + }; + p_MPs->parallel_for(setPos, "setMPPositions"); + pumipic::RecordTime("PolyMPO_setMPPositions", timer.seconds()); +} + +void polympo_getMPTgtPositions_f(MPMesh_ptr p_mpmesh, + const int nComps, + const int numMPs, + double* mpPositionsHost){ + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + PMT_ALWAYS_ASSERT(nComps == vec3d_nEntries); + PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); + //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + + auto mpPositions = p_MPs->getData(); + auto mpAppID = p_MPs->getData(); + Kokkos::View mpPositionsCopy("mpPositionsCopy",vec3d_nEntries,numMPs); + auto getPos = PS_LAMBDA(const int&, const int& mp, const int& mask){ + if(mask){ + mpPositionsCopy(0,mpAppID(mp)) = mpPositions(mp,0); + mpPositionsCopy(1,mpAppID(mp)) = mpPositions(mp,1); + mpPositionsCopy(2,mpAppID(mp)) = mpPositions(mp,2); + } + }; + p_MPs->parallel_for(getPos, "getMPPositions"); + kkDbl2dViewHostU arrayHost(mpPositionsHost,nComps,numMPs); + Kokkos::deep_copy(arrayHost, mpPositionsCopy); +} + + void polympo_setMPRotLatLon_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, @@ -404,6 +503,56 @@ void polympo_getMPRotLatLon_f(MPMesh_ptr p_mpmesh, Kokkos::deep_copy(arrayHost, mpRotLatLonCopy); } + +void polympo_setMPTgtRotLatLon_f(MPMesh_ptr p_mpmesh, + const int nComps, + const int numMPs, + const double* mpRotLatLonIn){ + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); + PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); + //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + + auto mpRotLatLon = p_MPs->getData(); + auto mpAppID = p_MPs->getData(); + kkViewHostU mpRotLatLonIn_h(mpRotLatLonIn,nComps,numMPs); + Kokkos::View mpRotLatLonIn_d("mpRotLatLonDevice",vec2d_nEntries,numMPs); + Kokkos::deep_copy(mpRotLatLonIn_d, mpRotLatLonIn_h); + auto setPos = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ + if(mask){ + mpRotLatLon(mp,0) = mpRotLatLonIn_d(0, mpAppID(mp)); + mpRotLatLon(mp,1) = mpRotLatLonIn_d(1, mpAppID(mp)); + } + }; + p_MPs->parallel_for(setPos, "setMPRotLatLon"); +} + +void polympo_getMPTgtRotLatLon_f(MPMesh_ptr p_mpmesh, + const int nComps, + const int numMPs, + double* mpRotLatLonHost){ + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); + PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); + //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + + auto mpRotLatLon = p_MPs->getData(); + auto mpAppID = p_MPs->getData(); + Kokkos::View mpRotLatLonCopy("mpRotLatLonCopy",vec2d_nEntries,numMPs); + auto getPos = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ + if(mask){ + mpRotLatLonCopy(0,mpAppID(mp)) = mpRotLatLon(mp,0); + mpRotLatLonCopy(1,mpAppID(mp)) = mpRotLatLon(mp,1); + } + }; + p_MPs->parallel_for(getPos, "getMPRotLatLon"); + kkDbl2dViewHostU arrayHost(mpRotLatLonHost,nComps,numMPs); + Kokkos::deep_copy(arrayHost, mpRotLatLonCopy); +} + + void polympo_setMPMass_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpMassIn) { Kokkos::Timer timer; checkMPMeshValid(p_mpmesh); @@ -1037,6 +1186,21 @@ bool polympo_push1P_f(MPMesh_ptr p_mpmesh){ return is_migrating; } +void polympo_push_ahead_f(MPMesh_ptr p_mpmesh){ + checkMPMeshValid(p_mpmesh); + ((polyMPO::MPMesh*)p_mpmesh) ->push_ahead(); +} + +void polympo_push_swap_f(MPMesh_ptr p_mpmesh){ + checkMPMeshValid(p_mpmesh); + ((polyMPO::MPMesh*)p_mpmesh) ->push_swap(); +} + +void polympo_push_swap_pos_f(MPMesh_ptr p_mpmesh){ + checkMPMeshValid(p_mpmesh); + ((polyMPO::MPMesh*)p_mpmesh) ->push_swap_pos(); +} + void polympo_push_f(MPMesh_ptr p_mpmesh){ checkMPMeshValid(p_mpmesh); ((polyMPO::MPMesh*)p_mpmesh) ->push(); diff --git a/src/pmpo_c.h b/src/pmpo_c.h index 3ab2331..cfbe628 100644 --- a/src/pmpo_c.h +++ b/src/pmpo_c.h @@ -21,7 +21,7 @@ void polympo_setMPICommunicator_f(MPMesh_ptr p_mpmesh, MPI_Fint fcomm); //MP info void polympo_createMPs_f(MPMesh_ptr p_mpmesh, const int numElms, const int numMPs, int* mpsPerElm, const int* mp2Elm, const int* isMPActive); void polympo_startRebuildMPs_f(MPMesh_ptr p_mpmesh, const int numMPs, const int* allTgtMpElmIn, const int* addedMPMask); -void polympo_startRebuildMPs_f2(MPMesh_ptr p_mpmesh, const int size1, const int* arg1, const int size2, const int size3, const int* arg2, const int* arg3); +void polympo_startRebuildMPs_f2(MPMesh_ptr p_mpmesh, const int size1, const int* arg1, const int size2, const int size3, int* arg2, int* arg3); int polympo_getMPCount_f(MPMesh_ptr p_mpmesh); @@ -30,14 +30,22 @@ void polympo_finishRebuildMPs_f(MPMesh_ptr p_mpmesh); void polympo_setAppIDFunc_f(MPMesh_ptr p_mpmesh, IntVoidFunc getNext, void* appIDs); void polympo_setMPASAppIDFunc_f(MPMesh_ptr p_mpmesh, VoidVoidFunc getMPASAppID, void* arg1, const int arg2, const int arg3); +void polympo_getMPTgtElmID_f(MPMesh_ptr p_mpmesh, const int numMPs, int* elmIDs); void polympo_getMPCurElmID_f(MPMesh_ptr p_mpmesh, const int numMPs, int* elmIDs); void polympo_setMPLatLonRotatedFlag_f(MPMesh_ptr p_mpmesh, const int isRotateFlag); //MP slices -void polympo_setMPPositions_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpPositionsIn, const int setTgtPos); +//Positions +void polympo_setMPPositions_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpPositionsIn); void polympo_getMPPositions_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, double* mpPositionsIn); +void polympo_setMPTgtPositions_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpPositionsIn); +void polympo_getMPTgtPositions_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, double* mpPositionsIn); +//LatLon void polympo_setMPRotLatLon_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpRotLatLonIn); void polympo_getMPRotLatLon_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, double* mpRotLatLonHost); +void polympo_setMPTgtRotLatLon_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpRotLatLonIn); +void polympo_getMPTgtRotLatLon_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, double* mpRotLatLonHost); + void polympo_setMPMass_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpMassIn); void polympo_getMPMass_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, double* mpMassHost); void polympo_setMPVel_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpVelIn); @@ -94,7 +102,10 @@ void polympo_getMeshDualTriangleArea_f(MPMesh_ptr p_mpmesh, const int nVertices, // calculations void polympo_push_f(MPMesh_ptr p_mpmesh); +void polympo_push_ahead_f(MPMesh_ptr p_mpmesh); bool polympo_push1P_f(MPMesh_ptr p_mpmesh); +void polympo_push_swap_f(MPMesh_ptr p_mpmesh); +void polympo_push_swap_pos_f(MPMesh_ptr p_mpmesh); // Reconstruction of variables from MPs to mesh vertices void polympo_setReconstructionOfMass_f(MPMesh_ptr p_mpmesh, const int order, const int meshEntType); diff --git a/src/pmpo_fortran.f90 b/src/pmpo_fortran.f90 index 4efe641..b05ce49 100644 --- a/src/pmpo_fortran.f90 +++ b/src/pmpo_fortran.f90 @@ -97,8 +97,8 @@ subroutine polympo_startRebuildMPs2(mpMesh, size1, arg1, size2, size3, arg2, arg type(c_ptr), intent(in), value :: arg1 integer(c_int), value :: size2 integer(c_int), value :: size3 - type(c_ptr), intent(in), value :: arg2 - type(c_ptr), intent(in), value :: arg3 + type(c_ptr), value :: arg2 + type(c_ptr), value :: arg3 end subroutine ! !--------------------------------------------------------------------------- @@ -156,6 +156,16 @@ subroutine polympo_getMPCurElmID(mpMesh, numMPs, array) & integer(c_int), value :: numMPs type(c_ptr), value :: array end subroutine + + + subroutine polympo_getMPTgtElmID(mpMesh, numMPs, array) & + bind(C, NAME='polympo_getMPTgtElmID_f') + use :: iso_c_binding + type(c_ptr), value :: mpMesh + integer(c_int), value :: numMPs + type(c_ptr), value :: array + end subroutine + ! !--------------------------------------------------------------------------- !> @brief set the mp lat lon is rotational or normal !> @param mpmesh(in/out) MPMesh object @@ -167,6 +177,8 @@ subroutine polympo_setMPLatLonRotatedFlag(mpMesh, isRotateFlag) & type(c_ptr), value :: mpMesh integer(c_int), value :: isRotateFlag end subroutine + + !--------------------------------------------------------------------------- !> @brief set the MP positions array from a host array !> @param mpmesh(in/out) MPMesh object @@ -174,13 +186,12 @@ subroutine polympo_setMPLatLonRotatedFlag(mpMesh, isRotateFlag) & !> @param numMPs(in) number of the MPs !> @param array(in) MP current position 2D array (3,numMPs), allocated by user on host !--------------------------------------------------------------------------- - subroutine polympo_setMPPositions(mpMesh, nComps, numMPs, array, setTgtPos) & + subroutine polympo_setMPPositions(mpMesh, nComps, numMPs, array) & bind(C, NAME='polympo_setMPPositions_f') use :: iso_c_binding type(c_ptr), value :: mpMesh integer(c_int), value :: nComps, numMPs type(c_ptr), value :: array - integer(c_int), value:: setTgtPos end subroutine !--------------------------------------------------------------------------- !> @brief get the MP positions array from a polympo array @@ -197,6 +208,37 @@ subroutine polympo_getMPPositions(mpMesh, nComps, numMPs, array) & integer(c_int), value :: nComps, numMPs type(c_ptr), value :: array end subroutine + !--------------------------------------------------------------------------- + !> @brief set the MP positions array from a host array + !> @param mpmesh(in/out) MPMesh object + !> @param nComps(in) number of components, should always be 3 + !> @param numMPs(in) number of the MPs + !> @param array(in) MP current position 2D array (3,numMPs), allocated by user on host + !--------------------------------------------------------------------------- + subroutine polympo_setMPTgtPositions(mpMesh, nComps, numMPs, array) & + bind(C, NAME='polympo_setMPTgtPositions_f') + use :: iso_c_binding + type(c_ptr), value :: mpMesh + integer(c_int), value :: nComps, numMPs + type(c_ptr), value :: array + end subroutine + !--------------------------------------------------------------------------- + !> @brief get the MP positions array from a polympo array + !> @param mpmesh(in/out) MPMesh object + !> @param nComps(in) number of components, should always be 3 + !> @param numMPs(in) number of the MPs + !> @param array(in/out) output MP current position 2D array (3,numMPs), + !> allocated by user + !--------------------------------------------------------------------------- + subroutine polympo_getMPTgtPositions(mpMesh, nComps, numMPs, array) & + bind(C, NAME='polympo_getMPTgtPositions_f') + use :: iso_c_binding + type(c_ptr), value :: mpMesh + integer(c_int), value :: nComps, numMPs + type(c_ptr), value :: array + end subroutine + + !--------------------------------------------------------------------------- !> @brief set the MP latitude and longtitude array from a host array !> @param mpmesh(in/out) MPMesh object @@ -227,6 +269,40 @@ subroutine polympo_getMPRotLatLon(mpMesh, nComps, numMPs, array) & integer(c_int), value :: nComps, numMPs type(c_ptr), value :: array end subroutine + !--------------------------------------------------------------------------- + !> @brief set the MP latitude and longtitude array from a host array + !> @param mpmesh(in/out) MPMesh object + !> @param nComps(in) number of components, should always be 2 + !> @param numMPs(in) number of the MPs + !> @param array(in) input MP current lat and lon 2D array (2,numMPs), + !> allocated by user + !--------------------------------------------------------------------------- + subroutine polympo_setMPTgtRotLatLon(mpMesh, nComps, numMPs, array) & + bind(C, NAME='polympo_setMPTgtRotLatLon_f') + use :: iso_c_binding + type(c_ptr), value :: mpMesh + integer(c_int), value :: nComps, numMPs + type(c_ptr), intent(in), value :: array + end subroutine + !--------------------------------------------------------------------------- + !> @brief get the MP latitude and longtitude array from a polympo array + !> @param mpmesh(in/out) MPMesh object + !> @param nComps(in) number of components, should always be 2 + !> @param numMPs(in) number of the MPs + !> @param array(in/out) output MP current lat and lon 2D array (2,numMPs), + !> allocated by user + !--------------------------------------------------------------------------- + subroutine polympo_getMPTgtRotLatLon(mpMesh, nComps, numMPs, array) & + bind(C, NAME='polympo_getMPTgtRotLatLon_f') + use :: iso_c_binding + type(c_ptr), value :: mpMesh + integer(c_int), value :: nComps, numMPs + type(c_ptr), value :: array + end subroutine + + + + !--------------------------------------------------------------------------- !> @brief set the Mass MP array from a host array !> @param mpmesh(in/out) MPMesh object @@ -768,6 +844,27 @@ logical function polympo_push1P(mpMesh) & type(c_ptr), value :: mpMesh end function + subroutine polympo_push_ahead(mpMesh) & + bind(C, NAME='polympo_push_ahead_f') + use :: iso_c_binding + type(c_ptr), value :: mpMesh + end subroutine + + subroutine polympo_push_swap(mpMesh) & + bind(C, NAME='polympo_push_swap_f') + use :: iso_c_binding + type(c_ptr), value :: mpMesh + end subroutine + + subroutine polympo_push_swap_pos(mpMesh) & + bind(C, NAME='polympo_push_swap_pos_f') + use :: iso_c_binding + type(c_ptr), value :: mpMesh + end subroutine + + + + !--------------------------------------------------------------------------- !> @brief start the reconstruction of MP Mass to Mesh Vertices !> @param mpmesh(in/out) MPMesh object diff --git a/src/pmpo_materialPoints.cpp b/src/pmpo_materialPoints.cpp index aca7f2e..2b05f92 100644 --- a/src/pmpo_materialPoints.cpp +++ b/src/pmpo_materialPoints.cpp @@ -8,11 +8,13 @@ template pumipic::MemberTypeViews createInternalMemberViews(int numMPs, View mp2elm, View mpAppID){ auto mpInfo = ps::createMemberViews(numMPs); auto mpCurElmPos_m = ps::getMemberView(mpInfo); + auto mpTgtElmPos_m = ps::getMemberView(mpInfo); auto mpAppID_m = ps::getMemberView(mpInfo); auto mpStatus_m = ps::getMemberView(mpInfo); auto policy = Kokkos::RangePolicy(typename MemSpace::execution_space(), 0, numMPs); Kokkos::parallel_for("setMPinfo", policy, KOKKOS_LAMBDA(int i) { mpCurElmPos_m(i) = mp2elm(i); + mpTgtElmPos_m(i) = -1; mpStatus_m(i) = MP_ACTIVE; mpAppID_m(i) = mpAppID(i); }); @@ -113,6 +115,22 @@ void MaterialPoints::finishRebuild() { ps::destroyViews(rebuildFields.addedSlices_h); ps::destroyViews(addedSlices_d); rebuildFields.ongoing = false; + + //Debug + /* + int rank; + MPI_Comm_rank(mpi_comm, &rank); + if (rank==0) return; + auto curr_elm=getData(); + auto tgt_elm =getData(); + auto mpAppID = getData(); + auto testElm = PS_LAMBDA(const int& e, const int& mp, const bool& mask) { + if (mask) { + printf("R1: finishRebuild AppID %d Curr %d Tgt %d e %d \n", mpAppID(mp), curr_elm(mp), tgt_elm(mp), e); + } + }; + parallel_for(testElm, "curr_elm"); + */ } MPI_Comm MaterialPoints::getMPIComm() { From a9e74ab5300e295914e8e8ab12fc965a3a87ef82 Mon Sep 17 00:00:00 2001 From: Nath Date: Tue, 20 May 2025 14:26:36 -0400 Subject: [PATCH 19/42] Removed print annd some cleaning --- src/pmpo_MPMesh.cpp | 45 ++------------------------------------------- src/pmpo_c.cpp | 8 -------- 2 files changed, 2 insertions(+), 51 deletions(-) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index f2fa0c6..df9f3bb 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -133,26 +133,18 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ MPI_Comm comm = p_MPs->getMPIComm(); int comm_rank; MPI_Comm_rank(comm, &comm_rank); - Kokkos::parallel_for("countProcess", numElms, KOKKOS_LAMBDA(const int iElm){ - int pp_id=elm2Process(iElm); - }); - + //Since Mesh is static print pnly for 1 time step if(printVTPIndex==0) { printVTP_mesh(comm_rank); } - //assert(cudaDeviceSynchronize()==cudaSuccess); - //MPI_Barrier(MPI_COMM_WORLD); Vec3dView history("positionHistory",numMPs); Vec3dView resultLeft("positionResult",numMPs); Vec3dView resultRight("positionResult",numMPs); Vec3dView mpTgtPosArray("positionTarget",numMPs); Kokkos::View counter("counter",1); - assert(cudaDeviceSynchronize() == cudaSuccess); - //printf("Rank %d Foo4 Begin\n", comm_rank); - auto CVTElmCalc = PS_LAMBDA(const int& elm, const int& mp, const int&mask){ Vec3d MP(mpPositions(mp,0),mpPositions(mp,1),mpPositions(mp,2)); if(mask){ @@ -190,12 +182,7 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ iElm = closestElm; } } - if(mpAppID(mp)==0 || mpAppID(mp)==191) - printf("Pos %.15e %.15e %.15e => %.15e %.15e %.15e\n", mpPositions(mp,0), mpPositions(mp,1), - mpPositions(mp,2), mpTgtPos(mp,0), mpTgtPos(mp,1), mpTgtPos(mp, 2)); - if(printVTPIndex>=0 && numMPs>0){ - //printf("Rank %d mp %d counter %d \n", comm_rank, mp, counter); double d1 = dx[0]; double d2 = dx[2]; double d3 = dx[3]; @@ -217,10 +204,6 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ }; p_MPs->parallel_for(CVTElmCalc,"CVTTrackingElmCenterBasedCalc"); - assert(cudaDeviceSynchronize() == cudaSuccess); - //printf("Rank %d Foo4 End\n", comm_rank); - - if(printVTPIndex>=0 && numMPs>0){ Vec3dView::HostMirror h_history = Kokkos::create_mirror_view(history); Vec3dView::HostMirror h_resultLeft = Kokkos::create_mirror_view(resultLeft); @@ -351,27 +334,20 @@ bool getAnyIsMigrating(MaterialPoints* p_MPs, bool isMigrating) { } void MPMesh::push_ahead(){ - static int count0=0; - std::cout<<"Push_ahead"<<" "<computeRotLatLonIncr(); sphericalInterpolation(*this); //Push the MPs p_MPs->updateRotLatLonAndXYZ2Tgt(p_mesh->getSphereRadius()); - count0 ++; } bool MPMesh::push1P(){ //Given target location find the new element or the last element in a partioned mesh //and the process it belongs to so that migration can be checked - static int count_p=0; - CVTTrackingElmCenterBased(); - + CVTTrackingElmCenterBased(); //From the above two inputs find if any particle needs to be migrated bool anyIsMigrating = getAnyIsMigrating(p_MPs, p_MPs->check_migrate()); - count_p=count_p+1; return anyIsMigrating; - } void MPMesh::push_swap(){ @@ -379,7 +355,6 @@ void MPMesh::push_swap(){ p_MPs->updateMPElmID(); } - void MPMesh::push_swap_pos(){ //current becomes target, target becomes -1 //Making read for next push_ahead @@ -390,9 +365,6 @@ void MPMesh::push_swap_pos(){ void MPMesh::push(){ - static int count=0; - std::cout<<"Push"<<" "<computeRotLatLonIncr(); @@ -422,19 +394,6 @@ void MPMesh::push(){ } while (anyIsMigrating); - count ++; - - volatile int i = 0; - char hostname[256]; - gethostname(hostname, sizeof(hostname)); - MPI_Comm comm = p_MPs->getMPIComm(); - int comm_rank; - MPI_Comm_rank(comm, &comm_rank); - if(count==480){ - printf("Rank %d PID %d on %s ready for attach\n", comm_rank, getpid(), hostname); - fflush(stdout); - sleep(100); - } pumipic::RecordTime("PolyMPO_push", timer.seconds()); } diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index ce7b7ba..5b0cba8 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -210,16 +210,9 @@ void polympo_startRebuildMPs_f2(MPMesh_ptr p_mpmesh, auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; int offset = p_MPs->getElmIDoffset(); - int rank; - auto mpi_comm=p_MPs->getMPIComm(); - MPI_Comm_rank(mpi_comm, &rank); - - printf("Rank %d to add %d and delete %d ptcls \n", rank, nMPs_add, nMPs_delete); for (int k=0; k Date: Tue, 20 May 2025 17:35:12 -0400 Subject: [PATCH 20/42] Some more redudancy removed like trying to get new AppID from MPAS --- src/pmpo_c.cpp | 14 +----- src/pmpo_c.h | 1 - src/pmpo_fortran.f90 | 18 -------- src/pmpo_materialPoints.cpp | 88 ++----------------------------------- src/pmpo_materialPoints.hpp | 12 ++--- 5 files changed, 8 insertions(+), 125 deletions(-) diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index 5b0cba8..71473fd 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -249,17 +249,7 @@ void polympo_setAppIDFunc_f(MPMesh_ptr p_mpmesh, IntVoidFunc getNext, void* appI checkMPMeshValid(p_mpmesh); auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; polyMPO::IntFunc getNextAppID = [getNext, appIDs]() { return getNext(appIDs); }; - //p_MPs->setAppIDFunc(getNextAppID); -} - -//arg1 is blockPtr, arg2 is blockSize and arg3 is nCells -void polympo_setMPASAppIDFunc_f(MPMesh_ptr p_mpmesh, VoidVoidFunc getMPASAppID, void*arg1, - const int arg2, const int arg3) { - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - std::function polyMPO_getMPASAppID = [getMPASAppID, arg1, arg2, arg3](int iCell) - { int iParticleNew=-1; getMPASAppID(arg1, arg2, arg3, iCell, iParticleNew); return iParticleNew;}; - p_MPs->setAppIDFunc(polyMPO_getMPASAppID); + p_MPs->setAppIDFunc(getNextAppID); } void polympo_getMPTgtElmID_f(MPMesh_ptr p_mpmesh, @@ -1263,7 +1253,7 @@ void polympo_setElmGlobal_f(MPMesh_ptr p_mpmesh, const int nCells, const int* ar PMT_ALWAYS_ASSERT(p_mesh->meshEditable()); Kokkos::View arrayHost("arrayHost", nCells); for (int i = 0; i < nCells; i++) { - arrayHost(i) = array[i] - 1; // Decrease each value by 1 + arrayHost(i) = array[i] - 1; // TODO right now elmID offset is set after MPs initialized } //check the size PMT_ALWAYS_ASSERT(nCells == p_mesh->getNumElements()); diff --git a/src/pmpo_c.h b/src/pmpo_c.h index cfbe628..ead18d6 100644 --- a/src/pmpo_c.h +++ b/src/pmpo_c.h @@ -28,7 +28,6 @@ int polympo_getMPCount_f(MPMesh_ptr p_mpmesh); void polympo_finishRebuildMPs_f(MPMesh_ptr p_mpmesh); void polympo_setAppIDFunc_f(MPMesh_ptr p_mpmesh, IntVoidFunc getNext, void* appIDs); -void polympo_setMPASAppIDFunc_f(MPMesh_ptr p_mpmesh, VoidVoidFunc getMPASAppID, void* arg1, const int arg2, const int arg3); void polympo_getMPTgtElmID_f(MPMesh_ptr p_mpmesh, const int numMPs, int* elmIDs); void polympo_getMPCurElmID_f(MPMesh_ptr p_mpmesh, const int numMPs, int* elmIDs); diff --git a/src/pmpo_fortran.f90 b/src/pmpo_fortran.f90 index b05ce49..694b6e0 100644 --- a/src/pmpo_fortran.f90 +++ b/src/pmpo_fortran.f90 @@ -125,24 +125,6 @@ subroutine polympo_setAppIDFunc(mpMesh, getNext, appIDs) & type(c_ptr), value :: appIDs end subroutine - !--------------------------------------------------------------------------- - !> @brief Stores pointer to appID data structure and a function to retrieve them used in migration - !> @param mpmesh(in/out) MPMesh object - !> @param getNext(in) Pointer to function that returns next App IDs - !> @param appIDs(in) Pointer to opaque data application data structure (that may contain all available app IDs) - !--------------------------------------------------------------------------- - subroutine polympo_setMPASAppIDFunc(mpMesh, getMPASAppID, & - arg1, arg2, arg3) & - bind(C, NAME='polympo_setMPASAppIDFunc_f') - use :: iso_c_binding - type(c_ptr), value :: mpMesh - type(c_funptr), value :: getMPASAppID - type(c_ptr), value :: arg1 - integer(c_int), intent(in), value :: arg2 - integer(c_int), intent(in), value :: arg3 - end subroutine - - !--------------------------------------------------------------------------- !> @brief get the current element ID MP array from a polympo array !> @param mpmesh(in/out) MPMesh object diff --git a/src/pmpo_materialPoints.cpp b/src/pmpo_materialPoints.cpp index 2b05f92..46fb41a 100644 --- a/src/pmpo_materialPoints.cpp +++ b/src/pmpo_materialPoints.cpp @@ -114,23 +114,7 @@ void MaterialPoints::finishRebuild() { updateMaxAppID(); ps::destroyViews(rebuildFields.addedSlices_h); ps::destroyViews(addedSlices_d); - rebuildFields.ongoing = false; - - //Debug - /* - int rank; - MPI_Comm_rank(mpi_comm, &rank); - if (rank==0) return; - auto curr_elm=getData(); - auto tgt_elm =getData(); - auto mpAppID = getData(); - auto testElm = PS_LAMBDA(const int& e, const int& mp, const bool& mask) { - if (mask) { - printf("R1: finishRebuild AppID %d Curr %d Tgt %d e %d \n", mpAppID(mp), curr_elm(mp), tgt_elm(mp), e); - } - }; - parallel_for(testElm, "curr_elm"); - */ + rebuildFields.ongoing = false; } MPI_Comm MaterialPoints::getMPIComm() { @@ -166,7 +150,6 @@ void MaterialPoints::migrate() { Kokkos::Timer timer; auto MPs2Elm = getData(); auto MPs2Proc = getData(); - auto mpAppID = getData(); IntView new_elem("new_elem", MPs->capacity()); IntView new_process("new_process", MPs->capacity()); @@ -177,76 +160,11 @@ void MaterialPoints::migrate() { if (mask) { new_elem(mp) = MPs2Elm(mp); new_process(mp) = MPs2Proc(mp); - if(rank!=new_process(mp)){ - printf("Particle %d in rank %d to be moved from %d %d \n", mpAppID(mp), rank, e, new_elem(mp) ); - mpAppID(mp)=-1; - } } }; parallel_for(setMigrationFields, "setMigrationFields"); MPs->migrate(new_elem, new_process); - //AS REBUILT, mpAppID needs to be be recalled - mpAppID = getData(); - //Count MPs that have -1 appID, so that we can count no of MPs received by a rank - Kokkos::View numReceivedMPs("numReceivedMPs", 1); - Kokkos::deep_copy(numReceivedMPs, 0); - auto countnewMPs = PS_LAMBDA(const int& e, const int& mp, const bool& mask) { - if(mask){ - if (mpAppID(mp) == -1) - Kokkos::atomic_add(&numReceivedMPs(0), 1); - } - }; - parallel_for(countnewMPs, "countReceivedPtcls"); - auto numReceivedMPs_host = Kokkos::create_mirror_view(numReceivedMPs); - Kokkos::deep_copy(numReceivedMPs_host, numReceivedMPs); - - //Another array that contains new element id of the the MPs that have migrated - //Array size is #elements received - Kokkos::View receivedMPs2Elm("ReceivedMPs2Elm", numReceivedMPs_host(0)); - Kokkos::View counter("counter", 1); - Kokkos::deep_copy(counter, 0); - auto set_new_elem = PS_LAMBDA(const int& e, const int& mp, const bool& mask) { - if(mask){ - if (mpAppID(mp) == -1){ - auto count_temp=Kokkos::atomic_fetch_add(&counter(0), 1); - receivedMPs2Elm(count_temp) = e; - } - } - }; - parallel_for(set_new_elem, "countReceivedPtcls"); - //Bring them to CPU so that elm_id can be passed and a new mpAppID can be found from CPU - auto receivedMPs2Elm_host = Kokkos::create_mirror_view(receivedMPs2Elm); - Kokkos::deep_copy(receivedMPs2Elm_host, receivedMPs2Elm); - auto counter_host = Kokkos::create_mirror_view(counter); - Kokkos::deep_copy(counter_host, counter); - assert(numReceivedMPs_host(0)==counter_host(0)); - if(counter_host(0)) - std::cout <<"Rank "< appIDs; - for(int i=0; i appIDs_host(appIDs.data(), appIDs.size()); - Kokkos::View appIDs_d("appIDsDevice", appIDs.size()); - Kokkos::deep_copy(appIDs_d, appIDs_host); - - //If the mpAppID is -1 assign a new mpAppID - Kokkos::deep_copy(counter, 0); - auto set_appID = PS_LAMBDA(const int& e, const int& mp, const bool& mask) { - if(mask){ - if (mpAppID(mp) == -1){ - auto count_temp=Kokkos::atomic_fetch_add(&counter(0), 1); - mpAppID(mp)=appIDs_d(count_temp)-1; - } - } - }; - parallel_for(set_appID, "setApplicationIDs"); - if (getOpMode() == polyMPO::MP_DEBUG) printf("Material point migration: %f\n", timer.seconds()); pumipic::RecordTime("PolyMPO_migrate", timer.seconds()); @@ -254,8 +172,8 @@ void MaterialPoints::migrate() { bool MaterialPoints::rebuildOngoing() { return rebuildFields.ongoing; } -void MaterialPoints::setAppIDFunc(IntIntFunc getAppIDIn) { getAppID = getAppIDIn; } +void MaterialPoints::setAppIDFunc(IntFunc getAppIDIn) { getAppID = getAppIDIn; } -int MaterialPoints::getNextAppID(int iElm) { return getAppID(iElm); } +int MaterialPoints::getNextAppID() { return getAppID(); } } diff --git a/src/pmpo_materialPoints.hpp b/src/pmpo_materialPoints.hpp index c2a037f..70c71dc 100644 --- a/src/pmpo_materialPoints.hpp +++ b/src/pmpo_materialPoints.hpp @@ -127,8 +127,7 @@ class MaterialPoints { bool isRotatedFlag = false; Operating_Mode operating_mode; RebuildHelper rebuildFields; - //IntFunc getAppID; - IntIntFunc getAppID; + IntFunc getAppID; MPI_Comm mpi_comm; public: @@ -157,20 +156,15 @@ class MaterialPoints { typename std::enable_if::type setRebuildMPSlice(mpSliceData mpSliceIn); - //void setAppIDFunc(IntFunc getAppIDIn); - void setAppIDFunc(IntIntFunc getAppIDIn); - - int getNextAppID(int iElm); + void setAppIDFunc(IntFunc getAppIDIn); + int getNextAppID(); void rebuild() { IntView tgtElm("tgtElm", MPs->capacity()); auto tgtMpElm = MPs->get(); - auto mpAppID = MPs->get(); auto setTgtElm = PS_LAMBDA(const int& e, const int& mp, const int& mask) { if(mask) { - auto app_id=mpAppID(mp); tgtElm(mp) = tgtMpElm(mp); - if(app_id==191) printf("Going from elm %d to %d \n", e, tgtMpElm(mp)); } }; ps::parallel_for(MPs, setTgtElm, "setTargetElement"); From 4b6930f709eb022e79052883b282dcaffa969cc8 Mon Sep 17 00:00:00 2001 From: Nath Date: Wed, 21 May 2025 12:12:30 -0400 Subject: [PATCH 21/42] All tests on remus pass --- src/pmpo_MPMesh.cpp | 29 ++++++++++------------------ src/pmpo_c.cpp | 2 -- src/pmpo_mesh.hpp | 3 +-- test/testFortranCreateRebuildMPs.f90 | 5 ++++- test/testFortranMPAdvection.f90 | 23 ++++++---------------- test/testFortranMPAppIDs.f90 | 15 +++++++++++--- test/testFortranMPReconstruction.f90 | 8 +++++--- test/testMPAppIDs.cpp | 2 +- 8 files changed, 39 insertions(+), 48 deletions(-) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index df9f3bb..d525b04 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -128,22 +128,16 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ auto MPs2Proc = p_MPs->getData(); auto elm2Process = p_mesh->getElm2Process(); auto elm2global = p_mesh->getElmGlobal(); - auto mpAppID = p_MPs->getData(); - MPI_Comm comm = p_MPs->getMPIComm(); - int comm_rank; - MPI_Comm_rank(comm, &comm_rank); - //Since Mesh is static print pnly for 1 time step - if(printVTPIndex==0) { - printVTP_mesh(comm_rank); + if(printVTPIndex>=0) { + printVTP_mesh(printVTPIndex); } Vec3dView history("positionHistory",numMPs); Vec3dView resultLeft("positionResult",numMPs); Vec3dView resultRight("positionResult",numMPs); Vec3dView mpTgtPosArray("positionTarget",numMPs); - Kokkos::View counter("counter",1); auto CVTElmCalc = PS_LAMBDA(const int& elm, const int& mp, const int&mask){ Vec3d MP(mpPositions(mp,0),mpPositions(mp,1),mpPositions(mp,2)); @@ -176,7 +170,8 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ if(closestElm<0){ MPs2Elm(mp) = iElm; - MPs2Proc(mp) = elm2Process(iElm); + if (elm2Process.size() > 0) + MPs2Proc(mp) = elm2Process(iElm); break; }else{ iElm = closestElm; @@ -194,31 +189,28 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ Vec3d shift = dx.cross(r) * ((1.0-0.7)*dx.magnitude()/(dx.cross(r)).magnitude()); Vec3d MPLeft = MParrow + shift; Vec3d MPRight = MParrow - shift; - auto xx=Kokkos::atomic_fetch_add(&counter(0), 1); - history(xx) = MP; - resultLeft(xx) = MPLeft; - resultRight(xx) = MPRight; - mpTgtPosArray(xx) = MPnew; + history(mp) = MP; + resultLeft(mp) = MPLeft; + resultRight(mp) = MPRight; + mpTgtPosArray(mp) = MPnew; } } }; p_MPs->parallel_for(CVTElmCalc,"CVTTrackingElmCenterBasedCalc"); - if(printVTPIndex>=0 && numMPs>0){ + if(printVTPIndex>=0){ Vec3dView::HostMirror h_history = Kokkos::create_mirror_view(history); Vec3dView::HostMirror h_resultLeft = Kokkos::create_mirror_view(resultLeft); Vec3dView::HostMirror h_resultRight = Kokkos::create_mirror_view(resultRight); Vec3dView::HostMirror h_mpTgtPos = Kokkos::create_mirror_view(mpTgtPosArray); - Kokkos::View::HostMirror h_counter = Kokkos::create_mirror_view(counter); Kokkos::deep_copy(h_history, history); Kokkos::deep_copy(h_resultLeft, resultLeft); Kokkos::deep_copy(h_resultRight, resultRight); Kokkos::deep_copy(h_mpTgtPos, mpTgtPosArray); - Kokkos::deep_copy(h_counter, counter); // printVTP file char* fileOutput = (char *)malloc(sizeof(char) * 256); - sprintf(fileOutput, "polyMPOCVTTrackingElmCenter_MPtracks_%d_%d.vtp", comm_rank, printVTPIndex); + sprintf(fileOutput, "polyMPOCVTTrackingElmCenter_MPtracks_%d.vtp", printVTPIndex); FILE * pFile = fopen(fileOutput,"w"); free(fileOutput); fprintf(pFile, "\n \n \n \n \n",numMPs*4,numMPs*2); @@ -378,7 +370,6 @@ void MPMesh::push(){ bool anyIsMigrating = false; do { CVTTrackingElmCenterBased(); // move to Tgt_XYZ - assert(cudaDeviceSynchronize() == cudaSuccess); p_MPs->updateMPSlice(); // Tgt_XYZ becomes Cur_XYZ p_MPs->updateMPSlice(); // Tgt becomes Cur diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index 71473fd..4ac7275 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -123,7 +123,6 @@ void polympo_createMPs_f(MPMesh_ptr p_mpmesh, } } auto elm2global = p_mesh->getElmGlobal(); - auto mpsPerElm_d = create_mirror_view_and_copy(mpsPerElm, numElms); auto active_mp2Elm_d = create_mirror_view_and_copy(active_mp2Elm.data(), numActiveMPs); auto active_mpIDs_d = create_mirror_view_and_copy(active_mpIDs.data(), numActiveMPs); @@ -1250,7 +1249,6 @@ void polympo_setOwningProc_f(MPMesh_ptr p_mpmesh, const int nCells, const int* a void polympo_setElmGlobal_f(MPMesh_ptr p_mpmesh, const int nCells, const int* array){ checkMPMeshValid(p_mpmesh); auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - PMT_ALWAYS_ASSERT(p_mesh->meshEditable()); Kokkos::View arrayHost("arrayHost", nCells); for (int i = 0; i < nCells; i++) { arrayHost(i) = array[i] - 1; // TODO right now elmID offset is set after MPs initialized diff --git a/src/pmpo_mesh.hpp b/src/pmpo_mesh.hpp index 0819da2..22e6ddd 100644 --- a/src/pmpo_mesh.hpp +++ b/src/pmpo_mesh.hpp @@ -163,8 +163,7 @@ class Mesh { void setOwningProc(IntView owningProc) {PMT_ALWAYS_ASSERT(meshEdit_); owningProc_ = owningProc; } - void setElmGlobal(IntView globalElm) {PMT_ALWAYS_ASSERT(meshEdit_); - globalElm_ = globalElm; } + void setElmGlobal(IntView globalElm) {globalElm_ = globalElm;} IntView getElmGlobal(); void computeRotLatLonIncr(); diff --git a/test/testFortranCreateRebuildMPs.f90 b/test/testFortranCreateRebuildMPs.f90 index 61ea24a..478708a 100644 --- a/test/testFortranCreateRebuildMPs.f90 +++ b/test/testFortranCreateRebuildMPs.f90 @@ -25,7 +25,7 @@ subroutine createMPsTest(mpMesh, nCells, numMPs, mp2Elm, isMPActive, mpPosition) real(kind=MPAS_RKIND) :: ptOne = 0.1_MPAS_RKIND integer, parameter :: MP_ACTIVE = 1 integer, parameter :: MP_INACTIVE = 0 - integer, dimension(:), pointer :: mpsPerElm, mp2Elm, isMPActive + integer, dimension(:), pointer :: mpsPerElm, mp2Elm, isMPActive, globalElms real(kind=MPAS_RKIND), dimension(:,:), pointer :: mpPosition isMPActive = MP_ACTIVE !no inactive MPs and some changed below @@ -42,11 +42,13 @@ subroutine createMPsTest(mpMesh, nCells, numMPs, mp2Elm, isMPActive, mpPosition) end do allocate(mpsPerElm(nCells)) + allocate(globalElms(nCells)) mpsPerElm = 1 !all elements have 1 MP and some changed below mpsPerElm(1) = 0 !1st element has 0 MPs mpsPerElm(2) = 2 !2nd element has 2 MPs mpsPerElm(3) = 2 !3rd element has 2 MPs + call polympo_setElmGlobal(mpMesh, nCells, c_loc(globalElms)) call polympo_createMPs(mpMesh,nCells,numMPs,c_loc(mpsPerElm),c_loc(mp2Elm),c_loc(isMPActive)) !set mp positions @@ -80,6 +82,7 @@ subroutine createMPsTest(mpMesh, nCells, numMPs, mp2Elm, isMPActive, mpPosition) !deallocate MP variables deallocate(mpsPerElm) + deallocate(globalElms) end subroutine subroutine rebuildMPsTests(mpMesh, numMPs, mp2Elm, isMPActive, mpPosition) diff --git a/test/testFortranMPAdvection.f90 b/test/testFortranMPAdvection.f90 index abdedf7..292d806 100644 --- a/test/testFortranMPAdvection.f90 +++ b/test/testFortranMPAdvection.f90 @@ -218,7 +218,7 @@ program main real(kind=MPAS_RKIND), dimension(:), pointer :: xCell, yCell, zCell integer, dimension(:,:), pointer :: verticesOnCell, cellsOnCell integer :: numMPs, numMPsCount, numPush - integer, dimension(:), pointer :: mpsPerElm, mp2Elm, isMPActive, mp2Elm_new + integer, dimension(:), pointer :: mpsPerElm, mp2Elm, isMPActive, mp2Elm_new, globalElms real(kind=MPAS_RKIND), dimension(:,:), pointer :: mpPosition, mpLatLon, mpPositions_new, mpLatLon_new integer, parameter :: MP_ACTIVE = 1 integer, parameter :: MP_INACTIVE = 0 @@ -284,6 +284,7 @@ program main allocate(lonCell(nCells)) allocate(mpsPerElm(nCells)) + allocate(globalElms(nCells)) allocate(mp2Elm(numMPs)) allocate(mp2Elm_new(numMPs)) @@ -301,6 +302,7 @@ program main mp2Elm(numMPsCount+1:numMPsCount+localNumMPs) = i mpsPerElm(i) = localNumMPs numMPsCount = numMPsCount + localNumMPs + globalElms(i)=i end do call assert(numMPsCount == numMPs, "num mps miscounted") @@ -356,24 +358,11 @@ program main end do call assert(numMPsCount == numMPs, "num mps miscounted") - + call polympo_setElmGlobal(mpMesh, nCells, c_loc(globalElms)) call polympo_createMPs(mpMesh,nCells,numMPs,c_loc(mpsPerElm),c_loc(mp2Elm),c_loc(isMPActive)) call polympo_setMPRotLatLon(mpMesh,2,numMPs,c_loc(mpLatLon)) call polympo_setMPPositions(mpMesh,3,numMPs,c_loc(mpPosition)) - - !Another advection test to test if material poins come back to the same position - call runAdvectionTest2(mpMesh, numPush, latVertex, lonVertex, nEdgesOnCell, verticesOnCell, nVertices, sphereRadius) - call polympo_getMPPositions(mpMesh, 3, numMPs, c_loc(mpPositions_new)) - call polympo_getMPRotLatLon(mpMesh, 2, numMPs, c_loc(mpLatLon_new)) - call polympo_getMPCurElmID(mpMesh, numMPS, c_loc(mp2Elm_new)) - - do i = 1, numMPs - if ( abs(mpLatLon_new(2,i)-mpLatLon(2,i)) > max_push_diff ) then - max_push_diff = abs(mpLatLon_new(2,i)-mpLatLon(2,i)) - end if - end do - call assert(max_push_diff.le.TOLERANCE_PUSH , "MPs donot come back check push!") - + if (testType == "API") then call runApiTest(mpMesh, numMPs, nVertices, nCells, numPush, mpLatLon, mpPosition, xVertex, yVertex, zVertex, latVertex) else if (testType == "MIGRATION") then @@ -412,7 +401,7 @@ program main deallocate(isMPActive) deallocate(mpPosition) deallocate(mpLatLon) - + deallocate(globalElms) stop end program diff --git a/test/testFortranMPAppIDs.f90 b/test/testFortranMPAppIDs.f90 index bcae9d7..bdcbc8e 100644 --- a/test/testFortranMPAppIDs.f90 +++ b/test/testFortranMPAppIDs.f90 @@ -40,7 +40,7 @@ subroutine testAppIDPointer(mpMesh) & integer :: setMeshOption, setMPOption integer :: ierr, self integer :: mpi_comm_handle = MPI_COMM_WORLD - integer, dimension(:), pointer :: mpsPerElm, mp2Elm, isMPActive + integer, dimension(:), pointer :: mpsPerElm, mp2Elm, isMPActive, globalElms type(c_ptr) :: mpMesh integer :: nCells, numMPs, appID integer, parameter :: MP_ACTIVE = 1 @@ -56,16 +56,19 @@ subroutine testAppIDPointer(mpMesh) & setMeshOption = 1 !create a hard coded planar test mesh setMPOption = 0 !create an empty set of MPs mpMesh = polympo_createMPMesh(setMeshOption, setMPOption) - numMPs = 1 allocate(mpsPerElm(nCells)) + allocate(globalElms(nCells)) allocate(mp2Elm(numMPs)) allocate(isMPActive(numMPs)) call polympo_getMeshNumElms(mpMesh, nCells) mpsPerElm = 1 mp2Elm = 1 isMPActive = MP_ACTIVE + !This is a dummy array of global Cell IDs for each mesh element + call polympo_setElmGlobal(mpMesh, nCells, c_loc(globalElms)) call polympo_createMPs(mpMesh, nCells, numMPs, c_loc(mpsPerElm), c_loc(mp2Elm), c_loc(isMPActive)) + print *, "Done here" call polympo_setMPICommunicator(mpMesh, mpi_comm_handle) ! Set function and opaque data structure(list/queue) used to retrieve appIDS call polympo_setAppIDFunc(mpMesh, c_funloc(GetAppID), c_loc(queue)); @@ -74,9 +77,15 @@ subroutine testAppIDPointer(mpMesh) & call testAppIDPointer(mpMesh) ! Clean Up + deallocate(mpsPerElm) + deallocate(globalElms) + deallocate(mp2Elm) + deallocate(isMPActive) + + call polympo_deleteMPMesh(mpMesh) call queue_destroy(queue) call polympo_finalize() call mpi_finalize(ierr) -end program \ No newline at end of file +end program diff --git a/test/testFortranMPReconstruction.f90 b/test/testFortranMPReconstruction.f90 index 03a4165..4a9e910 100644 --- a/test/testFortranMPReconstruction.f90 +++ b/test/testFortranMPReconstruction.f90 @@ -32,7 +32,7 @@ program main integer, dimension(:,:), pointer :: verticesOnCell, cellsOnCell integer :: numMPs, vID - integer, dimension(:), pointer :: mpsPerElm, mp2Elm, isMPActive + integer, dimension(:), pointer :: mpsPerElm, mp2Elm, isMPActive, globalElms real(kind=MPAS_RKIND), dimension(:,:), pointer :: mpPosition, mpLatLon real(kind=MPAS_RKIND), dimension(:,:), pointer :: mpMass, mpVel real(kind=MPAS_RKIND), dimension(:), pointer :: meshVtxMass, meshElmMass, meshVtxMass1 @@ -84,6 +84,7 @@ program main !createMPs numMPs = nCells allocate(mpsPerElm(nCells)) + allocate(globalElms(nCells)) allocate(mp2Elm(numMPs)) allocate(isMPActive(numMPs)) allocate(mpPosition(3,numMPs)) @@ -109,7 +110,8 @@ program main mpPosition(2,i) = yCell(i) mpPosition(3,i) = zCell(i) end do - + + call polympo_setElmGlobal(mpMesh, nCells, c_loc(globalElms)) call polympo_createMPs(mpMesh,nCells,numMPs,c_loc(mpsPerElm),c_loc(mp2Elm),c_loc(isMPActive)) call polympo_setMPICommunicator(mpMesh, mpi_comm_handle) call polympo_setMPRotLatLon(mpMesh,2,numMPs,c_loc(mpLatLon)) @@ -188,7 +190,7 @@ program main deallocate(meshElmMass) deallocate(meshVtxVelu) deallocate(meshVtxVelv) - + deallocate(globalElms) stop contains diff --git a/test/testMPAppIDs.cpp b/test/testMPAppIDs.cpp index 4d33658..c3064cc 100644 --- a/test/testMPAppIDs.cpp +++ b/test/testMPAppIDs.cpp @@ -67,4 +67,4 @@ void testAppIDPointer(MPMesh_ptr p_mpmesh) { PMT_ALWAYS_ASSERT(numAddedMPsAfter_h == numAddedMPs); } -#endif \ No newline at end of file +#endif From 00643ff10962e3f642072bdffa646ad73c6e3831 Mon Sep 17 00:00:00 2001 From: Nath Date: Wed, 21 May 2025 12:21:57 -0400 Subject: [PATCH 22/42] Removed print statement --- test/testFortranMPAppIDs.f90 | 1 - 1 file changed, 1 deletion(-) diff --git a/test/testFortranMPAppIDs.f90 b/test/testFortranMPAppIDs.f90 index bdcbc8e..a7cf7f2 100644 --- a/test/testFortranMPAppIDs.f90 +++ b/test/testFortranMPAppIDs.f90 @@ -68,7 +68,6 @@ subroutine testAppIDPointer(mpMesh) & !This is a dummy array of global Cell IDs for each mesh element call polympo_setElmGlobal(mpMesh, nCells, c_loc(globalElms)) call polympo_createMPs(mpMesh, nCells, numMPs, c_loc(mpsPerElm), c_loc(mp2Elm), c_loc(isMPActive)) - print *, "Done here" call polympo_setMPICommunicator(mpMesh, mpi_comm_handle) ! Set function and opaque data structure(list/queue) used to retrieve appIDS call polympo_setAppIDFunc(mpMesh, c_funloc(GetAppID), c_loc(queue)); From 14b0c7404cc535b09ddd12d8e082169b020a8ed7 Mon Sep 17 00:00:00 2001 From: Nath Date: Mon, 2 Jun 2025 21:00:58 -0400 Subject: [PATCH 23/42] Parallel-reconstruction using MPAS --- src/pmpo_MPMesh.hpp | 10 ++ src/pmpo_MPMesh_assembly.hpp | 218 ++++++++++++++++++++++++++++++++++- src/pmpo_c.cpp | 31 +++++ src/pmpo_c.h | 11 ++ src/pmpo_fortran.f90 | 25 ++++ 5 files changed, 290 insertions(+), 5 deletions(-) diff --git a/src/pmpo_MPMesh.hpp b/src/pmpo_MPMesh.hpp index 7b54afb..b813ad6 100644 --- a/src/pmpo_MPMesh.hpp +++ b/src/pmpo_MPMesh.hpp @@ -64,6 +64,16 @@ class MPMesh{ template void assemblyVtx1(); + void subAssemblyVtx1(int size1, int size2, double* array); + void subAssemblyCoeffs(int dim1, int dim2, double* m11, double* m12, double* m13, double* m14, + double* m22, double* m23, double* m24, + double* m33, double* m34, + double* m44); + void solveMatrixAndRegularize(int dim1, double* m11, double* m12, double* m13, double* m14, + double* m22, double* m23, double* m24, + double* m33, double* m34, + double* m44); + template void setReconstructSlice(int order, MeshFieldType type); void reconstructSlices(); diff --git a/src/pmpo_MPMesh_assembly.hpp b/src/pmpo_MPMesh_assembly.hpp index 69e02e6..39ab02e 100644 --- a/src/pmpo_MPMesh_assembly.hpp +++ b/src/pmpo_MPMesh_assembly.hpp @@ -145,13 +145,13 @@ void MPMesh::computeMatricesAndSolve(){ for (int k=1; kgetElm2VtxConn(); + int numVtx = p_mesh->getNumVertices(); + auto vtxCoords = p_mesh->getMeshField(); + auto elm2Process = p_mesh->getElm2Process(); + //Dual Element Area for Regularization + auto dual_triangle_area=p_mesh->getMeshField(); + //Material Points + calcBasis(); + auto weight = p_MPs->getData(); + auto mpPositions = p_MPs->getData(); + //Radius + double radius = 1.0; + if(p_mesh->getGeomType() == geom_spherical_surf) + radius=p_mesh->getSphereRadius(); + + MPI_Comm comm = p_MPs->getMPIComm(); + int comm_rank; + MPI_Comm_rank(comm, &comm_rank); + + + kkDbl2dViewHostU m11_h(m11, dim1, dim2); + kkDbl2dViewHostU m12_h(m12, dim1, dim2); + kkDbl2dViewHostU m13_h(m13, dim1, dim2); + kkDbl2dViewHostU m14_h(m14, dim1, dim2); + kkDbl2dViewHostU m22_h(m22, dim1, dim2); + kkDbl2dViewHostU m23_h(m23, dim1, dim2); + kkDbl2dViewHostU m24_h(m24, dim1, dim2); + kkDbl2dViewHostU m33_h(m33, dim1, dim2); + kkDbl2dViewHostU m34_h(m34, dim1, dim2); + kkDbl2dViewHostU m44_h(m44, dim1, dim2); + Kokkos::View m11_d("m11", dim1, dim2); + Kokkos::View m12_d("m12", dim1, dim2); + Kokkos::View m13_d("m13", dim1, dim2); + Kokkos::View m14_d("m14", dim1, dim2); + Kokkos::View m22_d("m22", dim1, dim2); + Kokkos::View m23_d("m23", dim1, dim2); + Kokkos::View m24_d("m23", dim1, dim2); + Kokkos::View m33_d("m33", dim1, dim2); + Kokkos::View m34_d("m34", dim1, dim2); + Kokkos::View m44_d("m34", dim1, dim2); + + + auto sub_assemble = PS_LAMBDA(const int& elm, const int& mp, const int& mask) { + if(mask && (elm2Process(elm)==comm_rank)) { //if material point is 'active'/'enabled' + int nVtxE = elm2VtxConn(elm,0); //number of vertices bounding the element + for(int i=0; iparallel_for(sub_assemble, "sub_assembly"); + + Kokkos::deep_copy(m11_h, m11_d); + Kokkos::deep_copy(m12_h, m12_d); + Kokkos::deep_copy(m13_h, m13_d); + Kokkos::deep_copy(m14_h, m14_d); + Kokkos::deep_copy(m22_h, m22_d); + Kokkos::deep_copy(m23_h, m23_d); + Kokkos::deep_copy(m24_h, m24_d); + Kokkos::deep_copy(m33_h, m33_d); + Kokkos::deep_copy(m34_h, m34_d); + Kokkos::deep_copy(m44_h, m44_d); + +} + +void MPMesh::solveMatrixAndRegularize( int dim1, double* m11, double* m12, double* m13, double* m14, + double* m22, double* m23, double* m24, + double* m33, double* m34, + double* m44){ + + std::cout<<__FUNCTION__<getMeshField(); + + kkViewHostU m11_h(m11, dim1); + kkViewHostU m12_h(m12, dim1); + kkViewHostU m13_h(m13, dim1); + kkViewHostU m14_h(m14, dim1); + kkViewHostU m22_h(m22, dim1); + kkViewHostU m23_h(m23, dim1); + kkViewHostU m24_h(m24, dim1); + kkViewHostU m33_h(m33, dim1); + kkViewHostU m34_h(m34, dim1); + kkViewHostU m44_h(m44, dim1); + + Kokkos::View m11_d("m11", dim1); + Kokkos::View m12_d("m12", dim1); + Kokkos::View m13_d("m13", dim1); + Kokkos::View m14_d("m14", dim1); + Kokkos::View m22_d("m22", dim1); + Kokkos::View m23_d("m23", dim1); + Kokkos::View m24_d("m24", dim1); + Kokkos::View m33_d("m33", dim1); + Kokkos::View m34_d("m34", dim1); + Kokkos::View m44_d("m44", dim1); + + Kokkos::deep_copy(m11_d, m11_h); + Kokkos::deep_copy(m12_d, m12_h); + Kokkos::deep_copy(m13_d, m13_h); + Kokkos::deep_copy(m14_d, m14_h); + Kokkos::deep_copy(m22_d, m22_h); + Kokkos::deep_copy(m23_d, m23_h); + Kokkos::deep_copy(m24_d, m24_h); + Kokkos::deep_copy(m33_d, m33_h); + Kokkos::deep_copy(m34_d, m34_h); + Kokkos::deep_copy(m44_d, m44_h); + + Kokkos::View VtxCoeffs("VtxCoeffs", dim1); + double radius=p_mesh->getSphereRadius(); + Kokkos::parallel_for("fill", dim1, KOKKOS_LAMBDA(const int vtx){ + Vec4d v0 = {m11_d(vtx), m12_d(vtx), m13_d(vtx), m14_d(vtx)}; + Vec4d v1 = {m12_d(vtx), m22_d(vtx), m23_d(vtx), m24_d(vtx)}; + Vec4d v2 = {m13_d(vtx), m23_d(vtx), m33_d(vtx), m34_d(vtx)}; + Vec4d v3 = {m14_d(vtx), m24_d(vtx), m34_d(vtx), m44_d(vtx)}; + //Matrix4d A = {v0,v1,v2,v3}; + Matrix4d A_regularized = {v0, v1, v2, v3}; + double coeff[vec4d_nEntries]={0.0, 0.0, 0.0, 0.0}; + CholeskySolve4d_UnitRHS(A_regularized, coeff); + + double mScale=sqrt(dual_triangle_area(vtx,0))/radius; + coeff[0]=coeff[0]*mScale*mScale; + coeff[1]=coeff[1]*mScale; + coeff[2]=coeff[2]*mScale; + coeff[3]=coeff[3]*mScale; + + for (int i=0; iprecomputedVtxCoeffs = VtxCoeffs; + +} + +void MPMesh::subAssemblyVtx1(int size1, int size2, double* array) { + Kokkos::Timer timer; + + auto VtxCoeffs=this->precomputedVtxCoeffs; + + //Mesh Information + auto elm2VtxConn = p_mesh->getElm2VtxConn(); + int numVtx = p_mesh->getNumVertices(); + auto vtxCoords = p_mesh->getMeshField(); + auto elm2Process = p_mesh->getElm2Process(); + + constexpr MaterialPointSlice mpfIndex = MPF_Mass; + auto mpData = p_MPs->getData(); + auto weight = p_MPs->getData(); + auto mpPositions = p_MPs->getData(); + + double radius=p_mesh->getSphereRadius(); + + kkDbl2dViewHostU arrayHost(array, size1, size2); + Kokkos::View array_d("reconstructedIceArea", size1, size2); + + + MPI_Comm comm = p_MPs->getMPIComm(); + int comm_rank; + MPI_Comm_rank(comm, &comm_rank); + + auto sub_assemble = PS_LAMBDA(const int& elm, const int& mp, const int& mask) { + if(mask && (elm2Process(elm)==comm_rank)) { + int nVtxE = elm2VtxConn(elm,0); //number of vertices bounding the element + for(int i=0; iparallel_for(sub_assemble, "sub_assembly"); + + Kokkos::deep_copy(arrayHost, array_d); + + //assert(cudaDeviceSynchronize()==cudaSuccess); + pumipic::RecordTime("PolyMPO_subAssembly", timer.seconds()); +} + + + template void MPMesh::assemblyVtx1() { Kokkos::Timer timer; diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index 4ac7275..662cb5d 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -1226,6 +1226,37 @@ void polympo_setReconstructionOfStress_f(MPMesh_ptr p_mpmesh, const int order, c (void)meshEntType; } + +//With MPI communication done via MPAS +void polympo_iceAreaSubAssembly_f(MPMesh_ptr p_mpmesh, int size1, int size2, double* array){ + static int count=0; + std::cout<<__FUNCTION__<subAssemblyVtx1(size1, size2, array); + count ++ ; +} +void polympo_subAssemblyCoeffs_f(MPMesh_ptr p_mpmesh, int dim1, int dim2, double* m11, double* m12, double* m13, double* m14, + double* m22, double* m23, double* m24, + double* m33, double* m34, + double* m44){ + checkMPMeshValid(p_mpmesh); + auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); + mpmesh->subAssemblyCoeffs(dim1, dim2, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44); +} + +void polympo_regularize_and_solve_matrix_f(MPMesh_ptr p_mpmesh, int dim1, double* m11, double* m12, double* m13, double* m14, + double* m22, double* m23, double* m24, + double* m33, double* m34, + double* m44){ + checkMPMeshValid(p_mpmesh); + auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); + mpmesh->solveMatrixAndRegularize(dim1, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44); + +} + + + void polympo_applyReconstruction_f(MPMesh_ptr p_mpmesh){ checkMPMeshValid(p_mpmesh); auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); diff --git a/src/pmpo_c.h b/src/pmpo_c.h index ead18d6..e6ae917 100644 --- a/src/pmpo_c.h +++ b/src/pmpo_c.h @@ -113,6 +113,17 @@ void polympo_setReconstructionOfStrainRate_f(MPMesh_ptr p_mpmesh, const int orde void polympo_setReconstructionOfStress_f(MPMesh_ptr p_mpmesh, const int order, const int meshEntType); void polympo_applyReconstruction_f(MPMesh_ptr p_mpmesh); +//Reconstruction using MPAS +void polympo_iceAreaSubAssembly_f(MPMesh_ptr p_mpmesh, int size1, int size2, double* array); +void polympo_subAssemblyCoeffs_f(MPMesh_ptr p_mpmesh, int dim1, int dim2, double* m11, double* m12, double* m13, double* m14, + double* m22, double* m23, double* m24, + double* m33, double* m34, + double* m44); +void polympo_regularize_and_solve_matrix_f(MPMesh_ptr p_mpmesh, int dim1, double* m11, double* m12, double* m13, double* m14, + double* m22, double* m23, double* m24, + double* m33, double* m34, + double* m44); + // Timing void polympo_enableTiming_f(); void polympo_summarizeTime_f(); diff --git a/src/pmpo_fortran.f90 b/src/pmpo_fortran.f90 index 694b6e0..36fb89f 100644 --- a/src/pmpo_fortran.f90 +++ b/src/pmpo_fortran.f90 @@ -899,6 +899,31 @@ subroutine polympo_setReconstructionOfStress(mpMesh, order, meshEntType) & type(c_ptr), value :: mpMesh integer(c_int), value :: order, meshEntType end subroutine + + + subroutine polympo_iceAreaSubAssembly(mpMesh, size1, size2, array) & + bind(C, NAME='polympo_iceAreaSubAssembly_f') + use :: iso_c_binding + type(c_ptr), value :: mpMesh + integer(c_int), value :: size1, size2 + type(c_ptr), value :: array + end subroutine + subroutine polympo_subAssemblyCoeffs(mpMesh, dim1, dim2, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44) & + bind(C, NAME='polympo_subAssemblyCoeffs_f') + use :: iso_c_binding + type(c_ptr), value :: mpMesh + integer(c_int), value :: dim1, dim2 + type(c_ptr), value :: m11, m12, m13, m14, m22, m23, m24, m33, m34, m44 + end subroutine + subroutine polympo_regularize_and_solve_matrix(mpMesh, dim1, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44) & + bind(C, NAME='polympo_regularize_and_solve_matrix_f') + use :: iso_c_binding + type(c_ptr), value :: mpMesh + integer(c_int), value :: dim1 + type(c_ptr), value :: m11, m12, m13, m14, m22, m23, m24, m33, m34, m44 + end subroutine + + !--------------------------------------------------------------------------- !> @brief directly call the reconstruct of the MP fields to mesh fields !> @param mpmesh(in/out) MPMesh object From 8ef0501a7c77bae980853aa3d6e0f086295579ee Mon Sep 17 00:00:00 2001 From: Nath Date: Tue, 3 Jun 2025 13:35:24 -0400 Subject: [PATCH 24/42] MaxAppID commentted out for multiple Proc --- src/pmpo_c.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index 662cb5d..91d6dda 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -540,7 +540,7 @@ void polympo_setMPMass_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; PMT_ALWAYS_ASSERT(nComps == 1); //TODO mp_sclr_t PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); auto mpMass = p_MPs->getData(); auto mpAppID = p_MPs->getData(); @@ -562,7 +562,7 @@ void polympo_getMPMass_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; PMT_ALWAYS_ASSERT(nComps == 1); //TODO mp_sclr_t PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); auto mpMass = p_MPs->getData(); auto mpAppID = p_MPs->getData(); From 10f27c56bf49e7ade72e3005a34d63cd920b2436 Mon Sep 17 00:00:00 2001 From: Nath Date: Thu, 26 Jun 2025 14:49:21 -0400 Subject: [PATCH 25/42] Parallel Reconstruction Templated --- src/pmpo_MPMesh.cpp | 5 ++- src/pmpo_MPMesh.hpp | 3 +- src/pmpo_MPMesh_assembly.hpp | 80 +++++++++++++++++++++--------------- src/pmpo_c.cpp | 26 +++++++++--- src/pmpo_c.h | 8 ++-- src/pmpo_fortran.f90 | 23 +++++++++-- src/pmpo_mesh.hpp | 3 ++ 7 files changed, 102 insertions(+), 46 deletions(-) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index d525b04..b17a6f7 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -116,7 +116,7 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ int numVtxs = p_mesh->getNumVertices(); int numElms = p_mesh->getNumElements(); auto numMPs = p_MPs->getCount(); - + const auto elmCenter = p_mesh->getMeshField(); auto elm2VtxConn = p_mesh->getElm2VtxConn(); @@ -167,6 +167,9 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ minDistSq = neighborDistSq; } } + if(abs(mpTgtPos(mp,0)+5.724649188625998e+06) < 1e-10) + printf("Track: %.15e %.15e %.15e => %.15e %.15e %.15e \n", mpTgtPos(mp,0), mpTgtPos(mp,1), mpTgtPos(mp,2), + mpPositions(mp,0), mpPositions(mp,1), mpPositions(mp,2) ); if(closestElm<0){ MPs2Elm(mp) = iElm; diff --git a/src/pmpo_MPMesh.hpp b/src/pmpo_MPMesh.hpp index b813ad6..238cce0 100644 --- a/src/pmpo_MPMesh.hpp +++ b/src/pmpo_MPMesh.hpp @@ -63,8 +63,9 @@ class MPMesh{ void assemblyElm0(); template void assemblyVtx1(); + template + void subAssemblyVtx1(int size1, int size2, double* array, int comp); - void subAssemblyVtx1(int size1, int size2, double* array); void subAssemblyCoeffs(int dim1, int dim2, double* m11, double* m12, double* m13, double* m14, double* m22, double* m23, double* m24, double* m33, double* m34, diff --git a/src/pmpo_MPMesh_assembly.hpp b/src/pmpo_MPMesh_assembly.hpp index 39ab02e..fdec9e7 100644 --- a/src/pmpo_MPMesh_assembly.hpp +++ b/src/pmpo_MPMesh_assembly.hpp @@ -103,6 +103,7 @@ void MPMesh::resetPreComputeFlag(){ isPreComputed = false; } +//Method 1 void MPMesh::computeMatricesAndSolve(){ Kokkos::Timer timer; //Mesh Information @@ -208,34 +209,38 @@ void MPMesh::computeMatricesAndSolve(){ pumipic::RecordTime("PolyMPO_Calculate_MLS_Coeff", timer.seconds()); } +//Method 2 void MPMesh::subAssemblyCoeffs(int dim1, int dim2, double* m11, double* m12, double* m13, double* m14, double* m22, double* m23, double* m24, double* m33, double* m34, double* m44){ - - std::cout<<__FUNCTION__<getMPIComm(); + int comm_rank; + MPI_Comm_rank(comm, &comm_rank); + //Mesh Information auto elm2VtxConn = p_mesh->getElm2VtxConn(); int numVtx = p_mesh->getNumVertices(); auto vtxCoords = p_mesh->getMeshField(); auto elm2Process = p_mesh->getElm2Process(); + auto elm2global = p_mesh->getElmGlobal(); + //Dual Element Area for Regularization auto dual_triangle_area=p_mesh->getMeshField(); + //Material Points calcBasis(); auto weight = p_MPs->getData(); - auto mpPositions = p_MPs->getData(); + auto mpPos = p_MPs->getData(); + auto mpAppID = p_MPs->getData(); + //Radius double radius = 1.0; if(p_mesh->getGeomType() == geom_spherical_surf) radius=p_mesh->getSphereRadius(); - MPI_Comm comm = p_MPs->getMPIComm(); - int comm_rank; - MPI_Comm_rank(comm, &comm_rank); - - kkDbl2dViewHostU m11_h(m11, dim1, dim2); kkDbl2dViewHostU m12_h(m12, dim1, dim2); kkDbl2dViewHostU m13_h(m13, dim1, dim2); @@ -257,7 +262,6 @@ void MPMesh::subAssemblyCoeffs(int dim1, int dim2, double* m11, double* m12, dou Kokkos::View m34_d("m34", dim1, dim2); Kokkos::View m44_d("m34", dim1, dim2); - auto sub_assemble = PS_LAMBDA(const int& elm, const int& mp, const int& mask) { if(mask && (elm2Process(elm)==comm_rank)) { //if material point is 'active'/'enabled' int nVtxE = elm2VtxConn(elm,0); //number of vertices bounding the element @@ -268,20 +272,20 @@ void MPMesh::subAssemblyCoeffs(int dim1, int dim2, double* m11, double* m12, dou double mScale=sqrt(dual_triangle_area(vID,0))/radius; Kokkos::atomic_add(&m11_d(i,elm), w_vtx*mScale*mScale); - Kokkos::atomic_add(&m12_d(i,elm), w_vtx*mScale*(vtxCoords(vID,0)-mpPositions(mp,0))/radius); - Kokkos::atomic_add(&m13_d(i,elm), w_vtx*mScale*(vtxCoords(vID,1)-mpPositions(mp,1))/radius); - Kokkos::atomic_add(&m14_d(i,elm), w_vtx*mScale*(vtxCoords(vID,2)-mpPositions(mp,2))/radius); - Kokkos::atomic_add(&m22_d(i,elm), w_vtx*mScale*(vtxCoords(vID,0)-mpPositions(mp,0))*(vtxCoords(vID,0)-mpPositions(mp,0))/(radius*radius)); - Kokkos::atomic_add(&m23_d(i,elm), w_vtx*mScale*(vtxCoords(vID,0)-mpPositions(mp,0))*(vtxCoords(vID,1)-mpPositions(mp,1))/(radius*radius)); - Kokkos::atomic_add(&m24_d(i,elm), w_vtx*mScale*(vtxCoords(vID,0)-mpPositions(mp,0))*(vtxCoords(vID,2)-mpPositions(mp,2))/(radius*radius)); - Kokkos::atomic_add(&m33_d(i,elm), w_vtx*mScale*(vtxCoords(vID,1)-mpPositions(mp,1))*(vtxCoords(vID,1)-mpPositions(mp,1))/(radius*radius)); - Kokkos::atomic_add(&m34_d(i,elm), w_vtx*mScale*(vtxCoords(vID,1)-mpPositions(mp,1))*(vtxCoords(vID,2)-mpPositions(mp,2))/(radius*radius)); - Kokkos::atomic_add(&m44_d(i,elm), w_vtx*mScale*(vtxCoords(vID,2)-mpPositions(mp,2))*(vtxCoords(vID,2)-mpPositions(mp,2))/(radius*radius)); + Kokkos::atomic_add(&m12_d(i,elm), w_vtx*mScale*(vtxCoords(vID,0)-mpPos(mp,0))/radius); + Kokkos::atomic_add(&m13_d(i,elm), w_vtx*mScale*(vtxCoords(vID,1)-mpPos(mp,1))/radius); + Kokkos::atomic_add(&m14_d(i,elm), w_vtx*mScale*(vtxCoords(vID,2)-mpPos(mp,2))/radius); + Kokkos::atomic_add(&m22_d(i,elm), w_vtx*mScale*(vtxCoords(vID,0)-mpPos(mp,0))*(vtxCoords(vID,0)-mpPos(mp,0))/(radius*radius)); + Kokkos::atomic_add(&m23_d(i,elm), w_vtx*mScale*(vtxCoords(vID,0)-mpPos(mp,0))*(vtxCoords(vID,1)-mpPos(mp,1))/(radius*radius)); + Kokkos::atomic_add(&m24_d(i,elm), w_vtx*mScale*(vtxCoords(vID,0)-mpPos(mp,0))*(vtxCoords(vID,2)-mpPos(mp,2))/(radius*radius)); + Kokkos::atomic_add(&m33_d(i,elm), w_vtx*mScale*(vtxCoords(vID,1)-mpPos(mp,1))*(vtxCoords(vID,1)-mpPos(mp,1))/(radius*radius)); + Kokkos::atomic_add(&m34_d(i,elm), w_vtx*mScale*(vtxCoords(vID,1)-mpPos(mp,1))*(vtxCoords(vID,2)-mpPos(mp,2))/(radius*radius)); + Kokkos::atomic_add(&m44_d(i,elm), w_vtx*mScale*(vtxCoords(vID,2)-mpPos(mp,2))*(vtxCoords(vID,2)-mpPos(mp,2))/(radius*radius)); } } }; p_MPs->parallel_for(sub_assemble, "sub_assembly"); - + Kokkos::deep_copy(m11_h, m11_d); Kokkos::deep_copy(m12_h, m12_d); Kokkos::deep_copy(m13_h, m13_d); @@ -295,12 +299,21 @@ void MPMesh::subAssemblyCoeffs(int dim1, int dim2, double* m11, double* m12, dou } +//Method 2 void MPMesh::solveMatrixAndRegularize( int dim1, double* m11, double* m12, double* m13, double* m14, double* m22, double* m23, double* m24, double* m33, double* m34, double* m44){ - std::cout<<__FUNCTION__<getMPIComm(); + int comm_rank; + MPI_Comm_rank(comm, &comm_rank); + + static int count=0; + if(!comm_rank) + std::cout<<__FUNCTION__<getMeshField(); kkViewHostU m11_h(m11, dim1); @@ -345,6 +358,9 @@ void MPMesh::solveMatrixAndRegularize( int dim1, double* m11, double* m12, doubl Vec4d v3 = {m14_d(vtx), m24_d(vtx), m34_d(vtx), m44_d(vtx)}; //Matrix4d A = {v0,v1,v2,v3}; Matrix4d A_regularized = {v0, v1, v2, v3}; + double regParam = sqrt(EPSILON)*(m11_d(vtx) + m22_d(vtx) + m33_d(vtx) + m44_d(vtx)); + A_regularized.addToDiag(regParam); + double coeff[vec4d_nEntries]={0.0, 0.0, 0.0, 0.0}; CholeskySolve4d_UnitRHS(A_regularized, coeff); @@ -361,10 +377,17 @@ void MPMesh::solveMatrixAndRegularize( int dim1, double* m11, double* m12, doubl } -void MPMesh::subAssemblyVtx1(int size1, int size2, double* array) { +//Method2 +template +void MPMesh::subAssemblyVtx1(int size1, int size2, double* array, int comp) { Kokkos::Timer timer; auto VtxCoeffs=this->precomputedVtxCoeffs; + + // MPI Information + MPI_Comm comm = p_MPs->getMPIComm(); + int comm_rank; + MPI_Comm_rank(comm, &comm_rank); //Mesh Information auto elm2VtxConn = p_mesh->getElm2VtxConn(); @@ -372,7 +395,8 @@ void MPMesh::subAssemblyVtx1(int size1, int size2, double* array) { auto vtxCoords = p_mesh->getMeshField(); auto elm2Process = p_mesh->getElm2Process(); - constexpr MaterialPointSlice mpfIndex = MPF_Mass; + // Material Points Information + constexpr MaterialPointSlice mpfIndex = meshFieldIndexToMPSlice; auto mpData = p_MPs->getData(); auto weight = p_MPs->getData(); auto mpPositions = p_MPs->getData(); @@ -382,11 +406,6 @@ void MPMesh::subAssemblyVtx1(int size1, int size2, double* array) { kkDbl2dViewHostU arrayHost(array, size1, size2); Kokkos::View array_d("reconstructedIceArea", size1, size2); - - MPI_Comm comm = p_MPs->getMPIComm(); - int comm_rank; - MPI_Comm_rank(comm, &comm_rank); - auto sub_assemble = PS_LAMBDA(const int& elm, const int& mp, const int& mask) { if(mask && (elm2Process(elm)==comm_rank)) { int nVtxE = elm2VtxConn(elm,0); //number of vertices bounding the element @@ -401,7 +420,7 @@ void MPMesh::subAssemblyVtx1(int size1, int size2, double* array) { VtxCoeffs(vID,2)*CoordDiffs[2] + VtxCoeffs(vID,3)*CoordDiffs[3]); - auto val = factor*mpData(mp,0); + auto val = factor*mpData(mp, comp); Kokkos::atomic_add(&array_d(i, elm), val); } } @@ -409,13 +428,10 @@ void MPMesh::subAssemblyVtx1(int size1, int size2, double* array) { p_MPs->parallel_for(sub_assemble, "sub_assembly"); Kokkos::deep_copy(arrayHost, array_d); - - //assert(cudaDeviceSynchronize()==cudaSuccess); pumipic::RecordTime("PolyMPO_subAssembly", timer.seconds()); } - - +//Method 1 template void MPMesh::assemblyVtx1() { Kokkos::Timer timer; diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index 91d6dda..83b6583 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -1228,13 +1228,15 @@ void polympo_setReconstructionOfStress_f(MPMesh_ptr p_mpmesh, const int order, c //With MPI communication done via MPAS -void polympo_iceAreaSubAssembly_f(MPMesh_ptr p_mpmesh, int size1, int size2, double* array){ - static int count=0; - std::cout<<__FUNCTION__<subAssemblyVtx1(size1, size2, array); - count ++ ; + mpmesh->subAssemblyVtx1(size1, size2, array, comp); +} +void polympo_vtxSubAssemblyVelocity_f(MPMesh_ptr p_mpmesh, int size1, int size2, double* array, int comp){ + checkMPMeshValid(p_mpmesh); + auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); + mpmesh->subAssemblyVtx1(size1, size2, array, comp); } void polympo_subAssemblyCoeffs_f(MPMesh_ptr p_mpmesh, int dim1, int dim2, double* m11, double* m12, double* m13, double* m14, double* m22, double* m23, double* m24, @@ -1292,6 +1294,20 @@ void polympo_setElmGlobal_f(MPMesh_ptr p_mpmesh, const int nCells, const int* ar p_mesh->setElmGlobal(elmGlobal); } +void polympo_setVtxGlobal_f(MPMesh_ptr p_mpmesh, const int nVertices, const int* array){ + checkMPMeshValid(p_mpmesh); + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + Kokkos::View arrayHost("arrayHost", nVertices); + for (int i = 0; i < nVertices; i++) { + arrayHost(i) = array[i] - 1; // TODO right now elmID offset is set after MPs initialized + } + //check the size + //PMT_ALWAYS_ASSERT(nVertices == p_mesh->getNumVertices()); + + Kokkos::View vtxGlobal("vtxGlobal",nVertices); + Kokkos::deep_copy(vtxGlobal, arrayHost); + p_mesh->setVtxGlobal(vtxGlobal); +} int polympo_getMPCount_f(MPMesh_ptr p_mpmesh) { auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; diff --git a/src/pmpo_c.h b/src/pmpo_c.h index e6ae917..6d9eda9 100644 --- a/src/pmpo_c.h +++ b/src/pmpo_c.h @@ -72,6 +72,7 @@ void polympo_setMeshElm2VtxConn_f(MPMesh_ptr p_mpmesh, const int maxEdges, const void polympo_setMeshElm2ElmConn_f(MPMesh_ptr p_mpmesh, const int maxEdges, const int nCells, const int* array); void polympo_setOwningProc_f(MPMesh_ptr p_mpmesh, const int nCells, const int* array); void polympo_setElmGlobal_f(MPMesh_ptr p_mpmesh, const int nCells, const int* array); +void polympo_setVtxGlobal_f(MPMesh_ptr p_mpmesh, const int nVertices, const int* array); //Mesh fields @@ -96,9 +97,7 @@ void polympo_getMeshElmCenter_f(MPMesh_ptr p_mpmesh, const int nCells, double* x //Area Triangle void polympo_setMeshDualTriangleArea_f(MPMesh_ptr p_mpmesh, const int nVertices, const double* areaTriangle); void polympo_getMeshDualTriangleArea_f(MPMesh_ptr p_mpmesh, const int nVertices, double* areaTriangle); - - - + // calculations void polympo_push_f(MPMesh_ptr p_mpmesh); void polympo_push_ahead_f(MPMesh_ptr p_mpmesh); @@ -114,7 +113,8 @@ void polympo_setReconstructionOfStress_f(MPMesh_ptr p_mpmesh, const int order, c void polympo_applyReconstruction_f(MPMesh_ptr p_mpmesh); //Reconstruction using MPAS -void polympo_iceAreaSubAssembly_f(MPMesh_ptr p_mpmesh, int size1, int size2, double* array); +void polympo_vtxSubAssemblyIceArea_f(MPMesh_ptr p_mpmesh, int size1, int size2, double* array, int comp); +void polympo_vtxSubAssemblyVelocity_f(MPMesh_ptr p_mpmesh, int size1, int size2, double* array, int comp); void polympo_subAssemblyCoeffs_f(MPMesh_ptr p_mpmesh, int dim1, int dim2, double* m11, double* m12, double* m13, double* m14, double* m22, double* m23, double* m24, double* m33, double* m34, diff --git a/src/pmpo_fortran.f90 b/src/pmpo_fortran.f90 index 36fb89f..7afb646 100644 --- a/src/pmpo_fortran.f90 +++ b/src/pmpo_fortran.f90 @@ -527,6 +527,7 @@ integer function polympo_getMeshFElmType() & bind(C, NAME='polympo_getMeshFElmType_f') use :: iso_c_binding end function + !--------------------------------------------------------------------------- !> @brief set the polympo mesh vertices coordinates !> @param mpmesh(in/out) MPMesh object @@ -803,7 +804,15 @@ subroutine polympo_setElmGlobal(mpMesh, nCells, array) & integer(c_int), value :: nCells type(c_ptr), intent(in), value :: array end subroutine + subroutine polympo_setVtxGlobal(mpMesh, nVertices, array) & + bind(C, NAME='polympo_setVtxGlobal_f') + use :: iso_c_binding + type(c_ptr), value :: mpMesh + integer(c_int), value :: nVertices + type(c_ptr), intent(in), value :: array + end subroutine + !--------------------------------------------------------------------------- !> @brief calculate the MPs from given mesh vertices rotational latitude @@ -901,13 +910,21 @@ subroutine polympo_setReconstructionOfStress(mpMesh, order, meshEntType) & end subroutine - subroutine polympo_iceAreaSubAssembly(mpMesh, size1, size2, array) & - bind(C, NAME='polympo_iceAreaSubAssembly_f') + subroutine polympo_vtxSubAssemblyIceArea(mpMesh, size1, size2, array, comp) & + bind(C, NAME='polympo_vtxSubAssemblyIceArea_f') + use :: iso_c_binding + type(c_ptr), value :: mpMesh + integer(c_int), value :: size1, size2, comp + type(c_ptr), value :: array + end subroutine + subroutine polympo_vtxSubAssemblyVelocity(mpMesh, size1, size2, array, comp) & + bind(C, NAME='polympo_vtxSubAssemblyVelocity_f') use :: iso_c_binding type(c_ptr), value :: mpMesh - integer(c_int), value :: size1, size2 + integer(c_int), value :: size1, size2, comp type(c_ptr), value :: array end subroutine + subroutine polympo_subAssemblyCoeffs(mpMesh, dim1, dim2, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44) & bind(C, NAME='polympo_subAssemblyCoeffs_f') use :: iso_c_binding diff --git a/src/pmpo_mesh.hpp b/src/pmpo_mesh.hpp index 22e6ddd..97910fe 100644 --- a/src/pmpo_mesh.hpp +++ b/src/pmpo_mesh.hpp @@ -89,6 +89,7 @@ class Mesh { IntElm2ElmView elm2ElmConn_; IntView owningProc_; IntView globalElm_; + IntView globalVtx_; //start of meshFields MeshFView vtxCoords_; MeshFView vtxRotLat_; @@ -165,6 +166,8 @@ class Mesh { void setElmGlobal(IntView globalElm) {globalElm_ = globalElm;} IntView getElmGlobal(); + void setVtxGlobal(IntView globalVtx) {globalVtx_ = globalVtx;} + IntView getVtxGlobal() {return globalVtx_;} void computeRotLatLonIncr(); }; From d4ac97a108e5d09fd81ad0301bde88d5ecb9db10 Mon Sep 17 00:00:00 2001 From: Nath Date: Tue, 1 Jul 2025 00:51:44 -0400 Subject: [PATCH 26/42] Some more clean up --- src/pmpo_MPMesh.cpp | 6 ------ src/pmpo_MPMesh_assembly.hpp | 17 ++++------------- src/pmpo_c.cpp | 8 -------- 3 files changed, 4 insertions(+), 27 deletions(-) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index b17a6f7..bf6c338 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -2,7 +2,6 @@ #include "pmpo_utils.hpp" #include "pmpo_MPMesh.hpp" #include "pmpo_wachspressBasis.hpp" -#include namespace polyMPO{ void printVTP_mesh(MPMesh& mpMesh, int printVTPIndex=-1); @@ -129,7 +128,6 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ auto elm2Process = p_mesh->getElm2Process(); auto elm2global = p_mesh->getElmGlobal(); - //Since Mesh is static print pnly for 1 time step if(printVTPIndex>=0) { printVTP_mesh(printVTPIndex); } @@ -167,10 +165,6 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ minDistSq = neighborDistSq; } } - if(abs(mpTgtPos(mp,0)+5.724649188625998e+06) < 1e-10) - printf("Track: %.15e %.15e %.15e => %.15e %.15e %.15e \n", mpTgtPos(mp,0), mpTgtPos(mp,1), mpTgtPos(mp,2), - mpPositions(mp,0), mpPositions(mp,1), mpPositions(mp,2) ); - if(closestElm<0){ MPs2Elm(mp) = iElm; if (elm2Process.size() > 0) diff --git a/src/pmpo_MPMesh_assembly.hpp b/src/pmpo_MPMesh_assembly.hpp index fdec9e7..7874e04 100644 --- a/src/pmpo_MPMesh_assembly.hpp +++ b/src/pmpo_MPMesh_assembly.hpp @@ -103,7 +103,7 @@ void MPMesh::resetPreComputeFlag(){ isPreComputed = false; } -//Method 1 +//Method 1 for coefficients void MPMesh::computeMatricesAndSolve(){ Kokkos::Timer timer; //Mesh Information @@ -209,7 +209,7 @@ void MPMesh::computeMatricesAndSolve(){ pumipic::RecordTime("PolyMPO_Calculate_MLS_Coeff", timer.seconds()); } -//Method 2 +//Method 2 for coefficients void MPMesh::subAssemblyCoeffs(int dim1, int dim2, double* m11, double* m12, double* m13, double* m14, double* m22, double* m23, double* m24, double* m33, double* m34, @@ -299,21 +299,12 @@ void MPMesh::subAssemblyCoeffs(int dim1, int dim2, double* m11, double* m12, dou } -//Method 2 -void MPMesh::solveMatrixAndRegularize( int dim1, double* m11, double* m12, double* m13, double* m14, +//Method 2 for coefficients Solve matrix +void MPMesh::solveMatrixAndRegularize(int dim1, double* m11, double* m12, double* m13, double* m14, double* m22, double* m23, double* m24, double* m33, double* m34, double* m44){ - MPI_Comm comm = p_MPs->getMPIComm(); - int comm_rank; - MPI_Comm_rank(comm, &comm_rank); - - static int count=0; - if(!comm_rank) - std::cout<<__FUNCTION__<getMeshField(); kkViewHostU m11_h(m11, dim1); diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index 83b6583..81cbf88 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -265,10 +265,6 @@ void polympo_getMPTgtElmID_f(MPMesh_ptr p_mpmesh, kkIntViewHostU arrayHost(elmIDs,numMPs); polyMPO::IntView mpTgtElmIDCopy("mpTgtElmIDNewValue",numMPs); - int rank; - auto mpi_comm=p_MPs->getMPIComm(); - MPI_Comm_rank(mpi_comm, &rank); - auto setTgtElmId = PS_LAMBDA(const int&, const int& mp, const int& mask){ if(mask){ mpTgtElmIDCopy(mpAppID(mp)) = mpTgtElmID(mp)+elmIDoffset; @@ -294,10 +290,6 @@ void polympo_getMPCurElmID_f(MPMesh_ptr p_mpmesh, kkIntViewHostU arrayHost(elmIDs,numMPs); polyMPO::IntView mpCurElmIDCopy("mpCurElmIDNewValue",numMPs); - int rank; - auto mpi_comm=p_MPs->getMPIComm(); - MPI_Comm_rank(mpi_comm, &rank); - auto getElmId = PS_LAMBDA(const int&, const int& mp, const int& mask){ if(mask){ mpCurElmIDCopy(mpAppID(mp)) = mpCurElmID(mp)+elmIDoffset; From b439df6a98f1c1fbe270caedb7c74ba7684742f9 Mon Sep 17 00:00:00 2001 From: Nath Date: Tue, 8 Jul 2025 14:45:25 -0400 Subject: [PATCH 27/42] Velocity Increment Interpolation and updating velocity in advection --- src/pmpo_MPMesh.cpp | 4 ++++ src/pmpo_MPMesh.hpp | 4 +++- src/pmpo_c.cpp | 4 ++-- src/pmpo_materialPoints.hpp | 13 +++++++++++-- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index bf6c338..71e6259 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -326,6 +326,10 @@ void MPMesh::push_ahead(){ //Latitude Longitude increment at mesh vertices and interpolate to particle position p_mesh->computeRotLatLonIncr(); sphericalInterpolation(*this); + //Interploate mesh velocity increments to particle positions + //Note that the basis fucntions are created twice and so need to avoid redeundant clualtions + //Tried template lists Template_Type... maybe better option available + sphericalInterpolation(*this); //Push the MPs p_MPs->updateRotLatLonAndXYZ2Tgt(p_mesh->getSphereRadius()); } diff --git a/src/pmpo_MPMesh.hpp b/src/pmpo_MPMesh.hpp index 238cce0..c280c36 100644 --- a/src/pmpo_MPMesh.hpp +++ b/src/pmpo_MPMesh.hpp @@ -7,11 +7,13 @@ namespace polyMPO{ -template const MaterialPointSlice meshFieldIndexToMPSlice; +template +const MaterialPointSlice meshFieldIndexToMPSlice; template <> const MaterialPointSlice meshFieldIndexToMPSlice < MeshF_Vel > = MPF_Vel; template <> const MaterialPointSlice meshFieldIndexToMPSlice < MeshF_VtxMass > = MPF_Mass; template <> const MaterialPointSlice meshFieldIndexToMPSlice < MeshF_ElmMass > = MPF_Mass; template <> const MaterialPointSlice meshFieldIndexToMPSlice < MeshF_RotLatLonIncr > = MPF_Rot_Lat_Lon_Incr; +template <> const MaterialPointSlice meshFieldIndexToMPSlice < MeshF_OnSurfVeloIncr > = MPF_Vel_Incr; #define maxMPsPerElm 8 diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index 81cbf88..90122b1 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -576,7 +576,7 @@ void polympo_setMPVel_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); auto mpVel = p_MPs->getData(); auto mpAppID = p_MPs->getData(); @@ -599,7 +599,7 @@ void polympo_getMPVel_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); auto mpVel = p_MPs->getData(); auto mpAppID = p_MPs->getData(); diff --git a/src/pmpo_materialPoints.hpp b/src/pmpo_materialPoints.hpp index 70c71dc..12ff130 100644 --- a/src/pmpo_materialPoints.hpp +++ b/src/pmpo_materialPoints.hpp @@ -36,6 +36,7 @@ enum MaterialPointSlice { MPF_Mass, MPF_Vel, MPF_Rot_Lat_Lon_Incr, + MPF_Vel_Incr, MPF_Strain_Rate, MPF_Stress, MPF_Stress_Div, @@ -64,6 +65,7 @@ template <> struct mpSliceToMeshField < MPF_Basis_Grad_Vals > { using type = template <> struct mpSliceToMeshField < MPF_Mass > { using type = doubleSclr_t; }; template <> struct mpSliceToMeshField < MPF_Vel > { using type = vec2d_t; }; template <> struct mpSliceToMeshField < MPF_Rot_Lat_Lon_Incr > { using type = vec2d_t; }; +template <> struct mpSliceToMeshField < MPF_Vel_Incr > { using type = vec2d_t; }; template <> struct mpSliceToMeshField < MPF_Strain_Rate > { using type = double[6]; }; template <> struct mpSliceToMeshField < MPF_Stress > { using type = double[6]; }; template <> struct mpSliceToMeshField < MPF_Stress_Div > { using type = vec3d_t; }; @@ -100,6 +102,7 @@ typedef MemberTypes::type, mpSliceToMeshField < MPF_Mass >::type, mpSliceToMeshField < MPF_Vel >::type, mpSliceToMeshField < MPF_Rot_Lat_Lon_Incr >::type, + mpSliceToMeshField < MPF_Vel_Incr >::type, mpSliceToMeshField < MPF_Strain_Rate >::type, mpSliceToMeshField < MPF_Stress >::type, mpSliceToMeshField < MPF_Stress_Div >::type, @@ -220,7 +223,10 @@ class MaterialPoints { auto tgtPosRotLatLon = MPs->get(); auto tgtPosXYZ = MPs->get(); auto rotLatLonIncr = MPs->get(); - + //Velocity + auto velMPs = MPs->get(); + auto velIncr = MPs->get(); + auto is_rotated = getRotatedFlag(); auto updateRotLatLon = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ if(mask){ @@ -235,10 +241,13 @@ class MaterialPoints { auto xyz_geo = grid_rotation_backward(xyz_rot); lat_lon_from_xyz(geoLat, geoLon, xyz_geo, radius); } - // x = cosLon cosLat, y = sinLon cosLat, z = sinLat + tgtPosXYZ(mp,0) = radius * std::cos(geoLon) * std::cos(geoLat); tgtPosXYZ(mp,1) = radius * std::sin(geoLon) * std::cos(geoLat); tgtPosXYZ(mp,2) = radius * std::sin(geoLat); + + velMPs(mp,0) = velMPs(mp,0) + velIncr(mp,0); + velMPs(mp,1) = velMPs(mp,1) + velIncr(mp,1); } }; ps::parallel_for(MPs, updateRotLatLon,"updateRotationalLatitudeLongitude"); From f89a03f07f52a1ec8e85a9ac20670f0716bef3bf Mon Sep 17 00:00:00 2001 From: Nath Date: Fri, 11 Jul 2025 09:48:54 -0400 Subject: [PATCH 28/42] Standard physics qualitative comparison, Kokkos fence before spherical interpolation of velocity increment, setget method of velocity increment with Timers --- src/pmpo_MPMesh.cpp | 1 + src/pmpo_c.cpp | 53 +++++++++++++++++++++++++++++---------------- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index 71e6259..6c08150 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -329,6 +329,7 @@ void MPMesh::push_ahead(){ //Interploate mesh velocity increments to particle positions //Note that the basis fucntions are created twice and so need to avoid redeundant clualtions //Tried template lists Template_Type... maybe better option available + Kokkos::fence(); sphericalInterpolation(*this); //Push the MPs p_MPs->updateRotLatLonAndXYZ2Tgt(p_mesh->getSphereRadius()); diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index 90122b1..71c8716 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -1075,11 +1075,16 @@ void polympo_getMeshElmMass_f(MPMesh_ptr p_mpmesh, const int nCells, double* elm pumipic::RecordTime("PolyMPO_getMeshElmMass", timer.seconds()); } +//Increments in vertex velcoity and displacement void polympo_setMeshVtxOnSurfVeloIncr_f(MPMesh_ptr p_mpmesh, const int nComps, const int nVertices, const double* array) { + + Kokkos::Timer timer; //check mpMesh is valid checkMPMeshValid(p_mpmesh); auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - kkViewHostU arrayHost(array,nVertices); + kkViewHostU arrayHost(array,nComps,nVertices); + Kokkos::View array_d("meshVelIncrDevice",nComps,nVertices); + Kokkos::deep_copy(array_d, arrayHost); auto vtxField = p_mesh->getMeshField(); @@ -1088,24 +1093,11 @@ void polympo_setMeshVtxOnSurfVeloIncr_f(MPMesh_ptr p_mpmesh, const int nComps, c PMT_ALWAYS_ASSERT(static_cast(nVertices*vec2d_nEntries)==vtxField.size()); //copy the host array to the device - Kokkos::deep_copy(vtxField,arrayHost); -} - -void polympo_getMeshVtxOnSurfVeloIncr_f(MPMesh_ptr p_mpmesh, const int nComps, const int nVertices, double* array) { - //check mpMesh is valid - checkMPMeshValid(p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - kkVec2dViewHostU arrayHost(array,nVertices); - - auto vtxField = p_mesh->getMeshField(); - - //check the size - PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); - PMT_ALWAYS_ASSERT(p_mesh->getNumVertices() == nVertices); - PMT_ALWAYS_ASSERT(static_cast(nVertices*vec2d_nEntries)==vtxField.size()); - - //copy the device array to the host - Kokkos::deep_copy(arrayHost, vtxField); + Kokkos::parallel_for("set mesh dispIncr", nVertices, KOKKOS_LAMBDA(const int iVtx){ + vtxField(iVtx,0) = array_d(0,iVtx); + vtxField(iVtx,1) = array_d(1,iVtx); + }); + pumipic::RecordTime("PolyMPO_setMeshVtxOnSurfVelIncr", timer.seconds()); } void polympo_setMeshVtxOnSurfDispIncr_f(MPMesh_ptr p_mpmesh, const int nComps, const int nVertices, const double* array) { @@ -1131,6 +1123,29 @@ void polympo_setMeshVtxOnSurfDispIncr_f(MPMesh_ptr p_mpmesh, const int nComps, c pumipic::RecordTime("PolyMPO_setMeshVtxOnSurfDispIncr", timer.seconds()); } + +void polympo_getMeshVtxOnSurfVeloIncr_f(MPMesh_ptr p_mpmesh, const int nComps, const int nVertices, double* array) { + //check mpMesh is valid + checkMPMeshValid(p_mpmesh); + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + kkDbl2dViewHostU arrayHost(array,nComps,nVertices); + Kokkos::View array_d("meshVelIncrDevice",nComps,nVertices); + + auto vtxField = p_mesh->getMeshField(); + + //check the size + PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); + PMT_ALWAYS_ASSERT(p_mesh->getNumVertices() == nVertices); + PMT_ALWAYS_ASSERT(static_cast(nVertices*vec2d_nEntries)==vtxField.size()); + + //copy the device array to the host + Kokkos::parallel_for("get mesh dispIncr", nVertices, KOKKOS_LAMBDA(const int iVtx){ + array_d(0,iVtx) = vtxField(iVtx,0); + array_d(1,iVtx) = vtxField(iVtx,1); + }); + Kokkos::deep_copy(arrayHost, array_d); +} + void polympo_getMeshVtxOnSurfDispIncr_f(MPMesh_ptr p_mpmesh, const int nComps, const int nVertices, double* array) { //check mpMesh is valid checkMPMeshValid(p_mpmesh); From d89991e7fb772f5cda63fcf4a545bef186907a85 Mon Sep 17 00:00:00 2001 From: Nath Date: Fri, 11 Jul 2025 15:23:46 -0400 Subject: [PATCH 29/42] Timings fixed/added --- src/pmpo_MPMesh.cpp | 4 ++++ src/pmpo_c.cpp | 15 ++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index 6c08150..64f19a6 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -323,6 +323,7 @@ bool getAnyIsMigrating(MaterialPoints* p_MPs, bool isMigrating) { } void MPMesh::push_ahead(){ + Kokkos::Timer timer; //Latitude Longitude increment at mesh vertices and interpolate to particle position p_mesh->computeRotLatLonIncr(); sphericalInterpolation(*this); @@ -333,14 +334,17 @@ void MPMesh::push_ahead(){ sphericalInterpolation(*this); //Push the MPs p_MPs->updateRotLatLonAndXYZ2Tgt(p_mesh->getSphereRadius()); + pumipic::RecordTime("PolyMPO_interpolateAndPush", timer.seconds()); } bool MPMesh::push1P(){ + Kokkos::Timer timer; //Given target location find the new element or the last element in a partioned mesh //and the process it belongs to so that migration can be checked CVTTrackingElmCenterBased(); //From the above two inputs find if any particle needs to be migrated bool anyIsMigrating = getAnyIsMigrating(p_MPs, p_MPs->check_migrate()); + pumipic::RecordTime("PolyMPO_trackAndCheckMigrate", timer.seconds()); return anyIsMigrating; } diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index 71c8716..fb38bda 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -254,6 +254,7 @@ void polympo_setAppIDFunc_f(MPMesh_ptr p_mpmesh, IntVoidFunc getNext, void* appI void polympo_getMPTgtElmID_f(MPMesh_ptr p_mpmesh, const int numMPs, int* elmIDs){ + Kokkos::Timer timer; checkMPMeshValid(p_mpmesh); auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); @@ -272,7 +273,7 @@ void polympo_getMPTgtElmID_f(MPMesh_ptr p_mpmesh, }; p_MPs->parallel_for(setTgtElmId, "set mpTgtElmID"); Kokkos::deep_copy( arrayHost, mpTgtElmIDCopy); - + pumipic::RecordTime("PolyMPO_getMPTgtElmID", timer.seconds()); } void polympo_getMPCurElmID_f(MPMesh_ptr p_mpmesh, @@ -396,13 +397,14 @@ void polympo_setMPTgtPositions_f(MPMesh_ptr p_mpmesh, } }; p_MPs->parallel_for(setPos, "setMPPositions"); - pumipic::RecordTime("PolyMPO_setMPPositions", timer.seconds()); + pumipic::RecordTime("PolyMPO_setMPTgtPositions", timer.seconds()); } void polympo_getMPTgtPositions_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, double* mpPositionsHost){ + Kokkos::Timer timer; checkMPMeshValid(p_mpmesh); auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; PMT_ALWAYS_ASSERT(nComps == vec3d_nEntries); @@ -422,6 +424,7 @@ void polympo_getMPTgtPositions_f(MPMesh_ptr p_mpmesh, p_MPs->parallel_for(getPos, "getMPPositions"); kkDbl2dViewHostU arrayHost(mpPositionsHost,nComps,numMPs); Kokkos::deep_copy(arrayHost, mpPositionsCopy); + pumipic::RecordTime("PolyMPO_getMPTgtPositions", timer.seconds()); } @@ -429,6 +432,7 @@ void polympo_setMPRotLatLon_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpRotLatLonIn){ + Kokkos::Timer timer; static int callCount = 0; PMT_ALWAYS_ASSERT(callCount == 0); checkMPMeshValid(p_mpmesh); @@ -450,6 +454,7 @@ void polympo_setMPRotLatLon_f(MPMesh_ptr p_mpmesh, }; p_MPs->parallel_for(setPos, "setMPRotLatLon"); callCount++; + pumipic::RecordTime("PolyMPO_setMPRotLatLon", timer.seconds()); } void polympo_getMPRotLatLon_f(MPMesh_ptr p_mpmesh, @@ -481,6 +486,7 @@ void polympo_setMPTgtRotLatLon_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpRotLatLonIn){ + Kokkos::Timer timer; checkMPMeshValid(p_mpmesh); auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); @@ -498,13 +504,15 @@ void polympo_setMPTgtRotLatLon_f(MPMesh_ptr p_mpmesh, mpRotLatLon(mp,1) = mpRotLatLonIn_d(1, mpAppID(mp)); } }; - p_MPs->parallel_for(setPos, "setMPRotLatLon"); + p_MPs->parallel_for(setPos, "setMPTgtRotLatLon"); + pumipic::RecordTime("PolyMPO_setMPTgtRotLatLon", timer.seconds()); } void polympo_getMPTgtRotLatLon_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, double* mpRotLatLonHost){ + Kokkos::Timer timer; checkMPMeshValid(p_mpmesh); auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); @@ -523,6 +531,7 @@ void polympo_getMPTgtRotLatLon_f(MPMesh_ptr p_mpmesh, p_MPs->parallel_for(getPos, "getMPRotLatLon"); kkDbl2dViewHostU arrayHost(mpRotLatLonHost,nComps,numMPs); Kokkos::deep_copy(arrayHost, mpRotLatLonCopy); + pumipic::RecordTime("PolyMPO_getMPTgtRotLatLon", timer.seconds()); } From 380ac706cf2ffc8f904b6bcb0c08ce1441b2ea42 Mon Sep 17 00:00:00 2001 From: Nath Date: Sun, 13 Jul 2025 10:03:56 -0400 Subject: [PATCH 30/42] Avoid calculating basis twice for 2 fields, this version in USNCCM18 performance --- src/pmpo_MPMesh.cpp | 10 ++++-- src/pmpo_wachspressBasis.hpp | 69 ++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 3 deletions(-) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index 64f19a6..9e97a8b 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -326,12 +326,16 @@ void MPMesh::push_ahead(){ Kokkos::Timer timer; //Latitude Longitude increment at mesh vertices and interpolate to particle position p_mesh->computeRotLatLonIncr(); - sphericalInterpolation(*this); + + //sphericalInterpolation(*this); //Interploate mesh velocity increments to particle positions //Note that the basis fucntions are created twice and so need to avoid redeundant clualtions //Tried template lists Template_Type... maybe better option available - Kokkos::fence(); - sphericalInterpolation(*this); + //Kokkos::fence(); + //sphericalInterpolation(*this); + + sphericalInterpolation1(*this); + //Push the MPs p_MPs->updateRotLatLonAndXYZ2Tgt(p_mesh->getSphereRadius()); pumipic::RecordTime("PolyMPO_interpolateAndPush", timer.seconds()); diff --git a/src/pmpo_wachspressBasis.hpp b/src/pmpo_wachspressBasis.hpp index cc3f49a..dfcbef5 100644 --- a/src/pmpo_wachspressBasis.hpp +++ b/src/pmpo_wachspressBasis.hpp @@ -358,5 +358,74 @@ void sphericalInterpolation(MPMesh& mpMesh){ pumipic::RecordTime("PolyMPO_sphericalInterpolation", timer.seconds()); } + +inline void sphericalInterpolation1(MPMesh& mpMesh){ + Kokkos::Timer timer; + auto p_mesh = mpMesh.p_mesh; + auto vtxCoords = p_mesh->getMeshField(); + int numVtxs = p_mesh->getNumVertices(); + auto elm2VtxConn = p_mesh->getElm2VtxConn(); + + auto p_MPs = mpMesh.p_MPs; + auto MPsPosition = p_MPs->getPositions(); + double radius = p_mesh->getSphereRadius(); + PMT_ALWAYS_ASSERT(radius > 0); + + constexpr MeshFieldIndex meshFieldIndex1 = polyMPO::MeshF_RotLatLonIncr; + constexpr MeshFieldIndex meshFieldIndex2 = polyMPO::MeshF_OnSurfVeloIncr; + + auto meshField1 = p_mesh->getMeshField(); + auto meshField2 = p_mesh->getMeshField(); + + constexpr MaterialPointSlice mpfIndex1 = meshFieldIndexToMPSlice; + constexpr MaterialPointSlice mpfIndex2 = meshFieldIndexToMPSlice; + + const int numEntries1 = mpSliceToNumEntries(); + const int numEntries2 = mpSliceToNumEntries(); + + auto mpField1 = p_MPs->getData(); + auto mpField2 = p_MPs->getData(); + + auto interpolation = PS_LAMBDA(const int& elm, const int& mp, const int& mask) { + if(mask) { + Vec3d position3d(MPsPosition(mp, 0), MPsPosition(mp, 1), MPsPosition(mp, 2)); + Vec3d v3d[maxVtxsPerElm + 1]; + int numVtx = elm2VtxConn(elm, 0); + for (int i = 1; i <= numVtx; i++) { + v3d[i-1][0] = vtxCoords(elm2VtxConn(elm, i) - 1, 0); + v3d[i-1][1] = vtxCoords(elm2VtxConn(elm, i) - 1, 1); + v3d[i-1][2] = vtxCoords(elm2VtxConn(elm, i) - 1, 2); + } + v3d[numVtx][0] = vtxCoords(elm2VtxConn(elm,1)-1,0); + v3d[numVtx][1] = vtxCoords(elm2VtxConn(elm,1)-1,1); + v3d[numVtx][2] = vtxCoords(elm2VtxConn(elm,1)-1,2); + + double basisByArea3d[maxVtxsPerElm] = {0.0}; + initArray(basisByArea3d, maxVtxsPerElm, 0.0); + + getBasisByAreaGblFormSpherical(position3d, numVtx, v3d, radius, basisByArea3d); + + for(int entry=0; entryparallel_for(interpolation, "sphericalInterpolationMultiField"); + pumipic::RecordTime("PolyMPO_sphericalInterpolation1", timer.seconds()); + } + + } //namespace polyMPO end #endif From 91d4185e25c33d7d18f72caf4d045d3098f6f01b Mon Sep 17 00:00:00 2001 From: dhyan1272 Date: Tue, 15 Jul 2025 12:47:09 -0700 Subject: [PATCH 31/42] Updated timers get set in reconstruction --- src/pmpo_MPMesh_assembly.hpp | 39 ++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/src/pmpo_MPMesh_assembly.hpp b/src/pmpo_MPMesh_assembly.hpp index 7874e04..1276d3b 100644 --- a/src/pmpo_MPMesh_assembly.hpp +++ b/src/pmpo_MPMesh_assembly.hpp @@ -215,6 +215,7 @@ void MPMesh::subAssemblyCoeffs(int dim1, int dim2, double* m11, double* m12, dou double* m33, double* m34, double* m44){ + Kokkos::Timer timer; //Material Points Information MPI_Comm comm = p_MPs->getMPIComm(); int comm_rank; @@ -241,16 +242,6 @@ void MPMesh::subAssemblyCoeffs(int dim1, int dim2, double* m11, double* m12, dou if(p_mesh->getGeomType() == geom_spherical_surf) radius=p_mesh->getSphereRadius(); - kkDbl2dViewHostU m11_h(m11, dim1, dim2); - kkDbl2dViewHostU m12_h(m12, dim1, dim2); - kkDbl2dViewHostU m13_h(m13, dim1, dim2); - kkDbl2dViewHostU m14_h(m14, dim1, dim2); - kkDbl2dViewHostU m22_h(m22, dim1, dim2); - kkDbl2dViewHostU m23_h(m23, dim1, dim2); - kkDbl2dViewHostU m24_h(m24, dim1, dim2); - kkDbl2dViewHostU m33_h(m33, dim1, dim2); - kkDbl2dViewHostU m34_h(m34, dim1, dim2); - kkDbl2dViewHostU m44_h(m44, dim1, dim2); Kokkos::View m11_d("m11", dim1, dim2); Kokkos::View m12_d("m12", dim1, dim2); Kokkos::View m13_d("m13", dim1, dim2); @@ -285,7 +276,20 @@ void MPMesh::subAssemblyCoeffs(int dim1, int dim2, double* m11, double* m12, dou } }; p_MPs->parallel_for(sub_assemble, "sub_assembly"); + pumipic::RecordTime("VtxSubAssemblyComputeCoeff", timer.seconds()); + Kokkos::Timer timer2; + kkDbl2dViewHostU m11_h(m11, dim1, dim2); + kkDbl2dViewHostU m12_h(m12, dim1, dim2); + kkDbl2dViewHostU m13_h(m13, dim1, dim2); + kkDbl2dViewHostU m14_h(m14, dim1, dim2); + kkDbl2dViewHostU m22_h(m22, dim1, dim2); + kkDbl2dViewHostU m23_h(m23, dim1, dim2); + kkDbl2dViewHostU m24_h(m24, dim1, dim2); + kkDbl2dViewHostU m33_h(m33, dim1, dim2); + kkDbl2dViewHostU m34_h(m34, dim1, dim2); + kkDbl2dViewHostU m44_h(m44, dim1, dim2); + Kokkos::deep_copy(m11_h, m11_d); Kokkos::deep_copy(m12_h, m12_d); Kokkos::deep_copy(m13_h, m13_d); @@ -296,6 +300,7 @@ void MPMesh::subAssemblyCoeffs(int dim1, int dim2, double* m11, double* m12, dou Kokkos::deep_copy(m33_h, m33_d); Kokkos::deep_copy(m34_h, m34_d); Kokkos::deep_copy(m44_h, m44_d); + pumipic::RecordTime("VtxSubAssemblyGetCoeff", timer2.seconds()); } @@ -305,8 +310,7 @@ void MPMesh::solveMatrixAndRegularize(int dim1, double* m11, double* m12, double double* m33, double* m34, double* m44){ - auto dual_triangle_area=p_mesh->getMeshField(); - + Kokkos::Timer timer; kkViewHostU m11_h(m11, dim1); kkViewHostU m12_h(m12, dim1); kkViewHostU m13_h(m13, dim1); @@ -339,7 +343,10 @@ void MPMesh::solveMatrixAndRegularize(int dim1, double* m11, double* m12, double Kokkos::deep_copy(m33_d, m33_h); Kokkos::deep_copy(m34_d, m34_h); Kokkos::deep_copy(m44_d, m44_h); + pumipic::RecordTime("polyMPOsolveMatrixCoeffSet", timer.seconds()); + Kokkos::Timer timer2; + auto dual_triangle_area=p_mesh->getMeshField(); Kokkos::View VtxCoeffs("VtxCoeffs", dim1); double radius=p_mesh->getSphereRadius(); Kokkos::parallel_for("fill", dim1, KOKKOS_LAMBDA(const int vtx){ @@ -365,6 +372,7 @@ void MPMesh::solveMatrixAndRegularize(int dim1, double* m11, double* m12, double VtxCoeffs(vtx,i)=coeff[i]; }); this->precomputedVtxCoeffs = VtxCoeffs; + pumipic::RecordTime("polyMPOsolveMatrixCoeffCompute", timer2.seconds()); } @@ -394,9 +402,7 @@ void MPMesh::subAssemblyVtx1(int size1, int size2, double* array, int comp) { double radius=p_mesh->getSphereRadius(); - kkDbl2dViewHostU arrayHost(array, size1, size2); Kokkos::View array_d("reconstructedIceArea", size1, size2); - auto sub_assemble = PS_LAMBDA(const int& elm, const int& mp, const int& mask) { if(mask && (elm2Process(elm)==comm_rank)) { int nVtxE = elm2VtxConn(elm,0); //number of vertices bounding the element @@ -417,9 +423,12 @@ void MPMesh::subAssemblyVtx1(int size1, int size2, double* array, int comp) { } }; p_MPs->parallel_for(sub_assemble, "sub_assembly"); + pumipic::RecordTime("polyMPOsubAssemblyFieldCompute", timer.seconds()); + Kokkos::Timer timer2; + kkDbl2dViewHostU arrayHost(array, size1, size2); Kokkos::deep_copy(arrayHost, array_d); - pumipic::RecordTime("PolyMPO_subAssembly", timer.seconds()); + pumipic::RecordTime("PolyMPOsubAssemblyFieldGet", timer2.seconds()); } //Method 1 From a786bc1b6f7d0b69b615eac877ead59ad148924f Mon Sep 17 00:00:00 2001 From: Nath Date: Wed, 16 Jul 2025 17:04:38 -0400 Subject: [PATCH 32/42] Couple of more timers --- src/pmpo_c.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index fb38bda..a9f956e 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -205,6 +205,8 @@ void polympo_startRebuildMPs_f2(MPMesh_ptr p_mpmesh, const int nMPs_add, int* recvMPs_elm, int* recvMPs_ids) { + + Kokkos::Timer timer; checkMPMeshValid(p_mpmesh); auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; int offset = p_MPs->getElmIDoffset(); @@ -233,15 +235,17 @@ void polympo_startRebuildMPs_f2(MPMesh_ptr p_mpmesh, int numDeletedMPs = pumipic::getLastValue(numDeletedMPs_d); assert(nMPs_delete==numDeletedMPs); p_MPs->startRebuild(mp2Elm, nMPs_add, recvMPs_elm_d, recvMPs_ids_d); + pumipic::RecordTime("polympo_startRebuildMPs_f2", timer.seconds()); } -void polympo_finishRebuildMPs_f(MPMesh_ptr p_mpmesh) -{ +void polympo_finishRebuildMPs_f(MPMesh_ptr p_mpmesh){ + Kokkos::Timer timer; checkMPMeshValid(p_mpmesh); auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; p_MPs->finishRebuild(); + pumipic::RecordTime("polympo_finishRebuildMPs_f", timer.seconds()); } void polympo_setAppIDFunc_f(MPMesh_ptr p_mpmesh, IntVoidFunc getNext, void* appIDs) { From 01d4525323ace4ae6c3cf7456e09db7c22839b49 Mon Sep 17 00:00:00 2001 From: Nath Date: Mon, 11 Aug 2025 14:10:15 -0400 Subject: [PATCH 33/42] Some more cleaning --- src/pmpo_MPMesh.cpp | 99 +++++++++++---------------------------------- 1 file changed, 23 insertions(+), 76 deletions(-) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index 9e97a8b..ec73534 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -128,15 +128,12 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ auto elm2Process = p_mesh->getElm2Process(); auto elm2global = p_mesh->getElmGlobal(); - if(printVTPIndex>=0) { + static int count_mesh_print=0; + if(printVTPIndex>=0 && count_mesh_print==0) { printVTP_mesh(printVTPIndex); + count_mesh_print += 1; } - Vec3dView history("positionHistory",numMPs); - Vec3dView resultLeft("positionResult",numMPs); - Vec3dView resultRight("positionResult",numMPs); - Vec3dView mpTgtPosArray("positionTarget",numMPs); - auto CVTElmCalc = PS_LAMBDA(const int& elm, const int& mp, const int&mask){ Vec3d MP(mpPositions(mp,0),mpPositions(mp,1),mpPositions(mp,2)); if(mask){ @@ -155,8 +152,7 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ for(int i=1; i<=numConnElms; i++){ int elmID = elm2ElmConn(iElm,i)-1; - //New delta - Vec3d center(elmCenter(elmID, 0), elmCenter(elmID, 1), elmCenter(elmID, 2)); + Vec3d center(elmCenter(elmID, 0), elmCenter(elmID, 1), elmCenter(elmID, 2)); delta = MPnew - center; double neighborDistSq = delta[0]*delta[0] + delta[1]*delta[1] + delta[2]*delta[2]; @@ -167,69 +163,16 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ } if(closestElm<0){ MPs2Elm(mp) = iElm; - if (elm2Process.size() > 0) - MPs2Proc(mp) = elm2Process(iElm); + MPs2Proc(mp) = elm2Process(iElm); break; }else{ iElm = closestElm; } } - if(printVTPIndex>=0 && numMPs>0){ - double d1 = dx[0]; - double d2 = dx[2]; - double d3 = dx[3]; - double m1 = MP[0]; - double m2 = MP[1]; - double m3 = MP[2]; - Vec3d MParrow = MP + dx*0.7; - Vec3d r = MPnew * (1.0/MPnew.magnitude()); - Vec3d shift = dx.cross(r) * ((1.0-0.7)*dx.magnitude()/(dx.cross(r)).magnitude()); - Vec3d MPLeft = MParrow + shift; - Vec3d MPRight = MParrow - shift; - history(mp) = MP; - resultLeft(mp) = MPLeft; - resultRight(mp) = MPRight; - mpTgtPosArray(mp) = MPnew; - } } }; p_MPs->parallel_for(CVTElmCalc,"CVTTrackingElmCenterBasedCalc"); - if(printVTPIndex>=0){ - Vec3dView::HostMirror h_history = Kokkos::create_mirror_view(history); - Vec3dView::HostMirror h_resultLeft = Kokkos::create_mirror_view(resultLeft); - Vec3dView::HostMirror h_resultRight = Kokkos::create_mirror_view(resultRight); - Vec3dView::HostMirror h_mpTgtPos = Kokkos::create_mirror_view(mpTgtPosArray); - - Kokkos::deep_copy(h_history, history); - Kokkos::deep_copy(h_resultLeft, resultLeft); - Kokkos::deep_copy(h_resultRight, resultRight); - Kokkos::deep_copy(h_mpTgtPos, mpTgtPosArray); - // printVTP file - char* fileOutput = (char *)malloc(sizeof(char) * 256); - sprintf(fileOutput, "polyMPOCVTTrackingElmCenter_MPtracks_%d.vtp", printVTPIndex); - FILE * pFile = fopen(fileOutput,"w"); - free(fileOutput); - fprintf(pFile, "\n \n \n \n \n",numMPs*4,numMPs*2); - for(int i=0; i\n \n \n \n"); - for(int i=0; i\n \n"); - for(int i=0; i\n \n \n \n\n"); - fclose(pFile); - } - pumipic::RecordTime("PolyMPO_CVTTrackingElmCenterBased", timer.seconds()); } @@ -322,49 +265,53 @@ bool getAnyIsMigrating(MaterialPoints* p_MPs, bool isMigrating) { return anyIsMigrating; } +//Spehrical Interpolation and Push void MPMesh::push_ahead(){ Kokkos::Timer timer; //Latitude Longitude increment at mesh vertices and interpolate to particle position p_mesh->computeRotLatLonIncr(); - //sphericalInterpolation(*this); - //Interploate mesh velocity increments to particle positions - //Note that the basis fucntions are created twice and so need to avoid redeundant clualtions - //Tried template lists Template_Type... maybe better option available - //Kokkos::fence(); - //sphericalInterpolation(*this); - + /* + sphericalInterpolation(*this); + Kokkos::fence(); + sphericalInterpolation(*this); + */ + + //The current spherical interpolation accepts just one template but for multiple + //fields the same weights can be used, maybe pass parameter list. Temporarily, + //application specific the following function sphericalInterpolation1(*this); - //Push the MPs + //Move the MPs p_MPs->updateRotLatLonAndXYZ2Tgt(p_mesh->getSphereRadius()); pumipic::RecordTime("PolyMPO_interpolateAndPush", timer.seconds()); } +//MP Tracking and migration check bool MPMesh::push1P(){ Kokkos::Timer timer; //Given target location find the new element or the last element in a partioned mesh - //and the process it belongs to so that migration can be checked + //for the MP. Also the owning process for that element. CVTTrackingElmCenterBased(); - //From the above two inputs find if any particle needs to be migrated + //From the above check if any particle needs to be migrated bool anyIsMigrating = getAnyIsMigrating(p_MPs, p_MPs->check_migrate()); pumipic::RecordTime("PolyMPO_trackAndCheckMigrate", timer.seconds()); return anyIsMigrating; } +//Current elm becomes the target elm, target elm becomes -1 void MPMesh::push_swap(){ - //current becomes target, target becomes -1 p_MPs->updateMPElmID(); } +//Current becomes the target, target becomes -1 +//Make read for next push_ahead void MPMesh::push_swap_pos(){ - //current becomes target, target becomes -1 - //Making read for next push_ahead p_MPs->updateMPSlice(); p_MPs->updateMPSlice(); } - +//Push routine where migration is carried out using polyMPO void MPMesh::push(){ Kokkos::Timer timer; From 9507b84e1564ad3c95fbe28df9180ab08b871bf1 Mon Sep 17 00:00:00 2001 From: Nath Date: Mon, 11 Aug 2025 14:31:33 -0400 Subject: [PATCH 34/42] elm2Process needed for ctest --- src/pmpo_MPMesh.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index ec73534..f5bc22a 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -163,7 +163,7 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ } if(closestElm<0){ MPs2Elm(mp) = iElm; - MPs2Proc(mp) = elm2Process(iElm); + if(elm2Process.size()>0) MPs2Proc(mp) = elm2Process(iElm); break; }else{ iElm = closestElm; From 14519cfd3d04c7de23552e309c1ea0f5afc9bfd4 Mon Sep 17 00:00:00 2001 From: Nath Date: Tue, 12 Aug 2025 17:43:18 -0400 Subject: [PATCH 35/42] RP changes1_v0 --- src/pmpo_MPMesh.cpp | 104 +++++++++++++++++++++++++---------- src/pmpo_MPMesh.hpp | 2 +- src/pmpo_MPMesh_assembly.hpp | 2 +- src/pmpo_c.cpp | 51 +++++++++-------- src/pmpo_c.h | 6 +- src/pmpo_fortran.f90 | 7 ++- src/pmpo_materialPoints.cpp | 2 +- src/pmpo_materialPoints.hpp | 4 +- src/pmpo_utils.hpp | 70 +++++++++++------------ 9 files changed, 149 insertions(+), 99 deletions(-) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index f5bc22a..5a1f9b5 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -115,7 +115,7 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ int numVtxs = p_mesh->getNumVertices(); int numElms = p_mesh->getNumElements(); auto numMPs = p_MPs->getCount(); - + const auto elmCenter = p_mesh->getMeshField(); auto elm2VtxConn = p_mesh->getElm2VtxConn(); @@ -128,12 +128,15 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ auto elm2Process = p_mesh->getElm2Process(); auto elm2global = p_mesh->getElmGlobal(); - static int count_mesh_print=0; - if(printVTPIndex>=0 && count_mesh_print==0) { + if(printVTPIndex>=0) { printVTP_mesh(printVTPIndex); - count_mesh_print += 1; } - + + Vec3dView history("positionHistory",numMPs); + Vec3dView resultLeft("positionResult",numMPs); + Vec3dView resultRight("positionResult",numMPs); + Vec3dView mpTgtPosArray("positionTarget",numMPs); + auto CVTElmCalc = PS_LAMBDA(const int& elm, const int& mp, const int&mask){ Vec3d MP(mpPositions(mp,0),mpPositions(mp,1),mpPositions(mp,2)); if(mask){ @@ -152,7 +155,8 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ for(int i=1; i<=numConnElms; i++){ int elmID = elm2ElmConn(iElm,i)-1; - Vec3d center(elmCenter(elmID, 0), elmCenter(elmID, 1), elmCenter(elmID, 2)); + //New delta + Vec3d center(elmCenter(elmID, 0), elmCenter(elmID, 1), elmCenter(elmID, 2)); delta = MPnew - center; double neighborDistSq = delta[0]*delta[0] + delta[1]*delta[1] + delta[2]*delta[2]; @@ -163,16 +167,69 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ } if(closestElm<0){ MPs2Elm(mp) = iElm; - if(elm2Process.size()>0) MPs2Proc(mp) = elm2Process(iElm); + if (elm2Process.size() > 0) + MPs2Proc(mp) = elm2Process(iElm); break; }else{ iElm = closestElm; } } + if(printVTPIndex>=0 && numMPs>0){ + double d1 = dx[0]; + double d2 = dx[2]; + double d3 = dx[3]; + double m1 = MP[0]; + double m2 = MP[1]; + double m3 = MP[2]; + Vec3d MParrow = MP + dx*0.7; + Vec3d r = MPnew * (1.0/MPnew.magnitude()); + Vec3d shift = dx.cross(r) * ((1.0-0.7)*dx.magnitude()/(dx.cross(r)).magnitude()); + Vec3d MPLeft = MParrow + shift; + Vec3d MPRight = MParrow - shift; + history(mp) = MP; + resultLeft(mp) = MPLeft; + resultRight(mp) = MPRight; + mpTgtPosArray(mp) = MPnew; + } } }; p_MPs->parallel_for(CVTElmCalc,"CVTTrackingElmCenterBasedCalc"); - + + if(printVTPIndex>=0){ + Vec3dView::HostMirror h_history = Kokkos::create_mirror_view(history); + Vec3dView::HostMirror h_resultLeft = Kokkos::create_mirror_view(resultLeft); + Vec3dView::HostMirror h_resultRight = Kokkos::create_mirror_view(resultRight); + Vec3dView::HostMirror h_mpTgtPos = Kokkos::create_mirror_view(mpTgtPosArray); + + Kokkos::deep_copy(h_history, history); + Kokkos::deep_copy(h_resultLeft, resultLeft); + Kokkos::deep_copy(h_resultRight, resultRight); + Kokkos::deep_copy(h_mpTgtPos, mpTgtPosArray); + // printVTP file + char* fileOutput = (char *)malloc(sizeof(char) * 256); + sprintf(fileOutput, "polyMPOCVTTrackingElmCenter_MPtracks_%d.vtp", printVTPIndex); + FILE * pFile = fopen(fileOutput,"w"); + free(fileOutput); + fprintf(pFile, "\n \n \n \n \n",numMPs*4,numMPs*2); + for(int i=0; i\n \n \n \n"); + for(int i=0; i\n \n"); + for(int i=0; i\n \n \n \n\n"); + fclose(pFile); + } + pumipic::RecordTime("PolyMPO_CVTTrackingElmCenterBased", timer.seconds()); } @@ -265,53 +322,44 @@ bool getAnyIsMigrating(MaterialPoints* p_MPs, bool isMigrating) { return anyIsMigrating; } -//Spehrical Interpolation and Push void MPMesh::push_ahead(){ Kokkos::Timer timer; //Latitude Longitude increment at mesh vertices and interpolate to particle position - p_mesh->computeRotLatLonIncr(); - - /* - sphericalInterpolation(*this); - Kokkos::fence(); - sphericalInterpolation(*this); - */ + p_mesh->computeRotLatLonIncr(); - //The current spherical interpolation accepts just one template but for multiple - //fields the same weights can be used, maybe pass parameter list. Temporarily, - //application specific the following function + //Interpolates latitude longitude increments and mesh velocity increments to + //MP positions sphericalInterpolation1(*this); - //Move the MPs + //Push the MPs p_MPs->updateRotLatLonAndXYZ2Tgt(p_mesh->getSphereRadius()); pumipic::RecordTime("PolyMPO_interpolateAndPush", timer.seconds()); } -//MP Tracking and migration check bool MPMesh::push1P(){ Kokkos::Timer timer; //Given target location find the new element or the last element in a partioned mesh - //for the MP. Also the owning process for that element. + //and the process it belongs to so that migration can be checked CVTTrackingElmCenterBased(); - //From the above check if any particle needs to be migrated + //From the above two inputs find if any particle needs to be migrated bool anyIsMigrating = getAnyIsMigrating(p_MPs, p_MPs->check_migrate()); pumipic::RecordTime("PolyMPO_trackAndCheckMigrate", timer.seconds()); return anyIsMigrating; } -//Current elm becomes the target elm, target elm becomes -1 void MPMesh::push_swap(){ + //current becomes target, target becomes -1 p_MPs->updateMPElmID(); } -//Current becomes the target, target becomes -1 -//Make read for next push_ahead void MPMesh::push_swap_pos(){ + //current becomes target, target becomes -1 + //Making read for next push_ahead p_MPs->updateMPSlice(); p_MPs->updateMPSlice(); } -//Push routine where migration is carried out using polyMPO + void MPMesh::push(){ Kokkos::Timer timer; @@ -341,7 +389,7 @@ void MPMesh::push(){ reconstructSlices(); } while (anyIsMigrating); - + pumipic::RecordTime("PolyMPO_push", timer.seconds()); } diff --git a/src/pmpo_MPMesh.hpp b/src/pmpo_MPMesh.hpp index c280c36..304833f 100644 --- a/src/pmpo_MPMesh.hpp +++ b/src/pmpo_MPMesh.hpp @@ -66,7 +66,7 @@ class MPMesh{ template void assemblyVtx1(); template - void subAssemblyVtx1(int size1, int size2, double* array, int comp); + void subAssemblyVtx1(int size1, int size2, int comp, double* array); void subAssemblyCoeffs(int dim1, int dim2, double* m11, double* m12, double* m13, double* m14, double* m22, double* m23, double* m24, diff --git a/src/pmpo_MPMesh_assembly.hpp b/src/pmpo_MPMesh_assembly.hpp index 1276d3b..ae1cf70 100644 --- a/src/pmpo_MPMesh_assembly.hpp +++ b/src/pmpo_MPMesh_assembly.hpp @@ -378,7 +378,7 @@ void MPMesh::solveMatrixAndRegularize(int dim1, double* m11, double* m12, double //Method2 template -void MPMesh::subAssemblyVtx1(int size1, int size2, double* array, int comp) { +void MPMesh::subAssemblyVtx1(int size1, int size2, int comp, double* array) { Kokkos::Timer timer; auto VtxCoeffs=this->precomputedVtxCoeffs; diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index a9f956e..92fa3fc 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -110,7 +110,6 @@ void polympo_createMPs_f(MPMesh_ptr p_mpmesh, fprintf(stderr,"The minElmID is incorrect! Offset is wrong!\n"); exit(1); } - printf("Offset %d \n", offset); std::vector active_mpIDs(numMPs); std::vector active_mp2Elm(numMPs); @@ -126,7 +125,7 @@ void polympo_createMPs_f(MPMesh_ptr p_mpmesh, auto mpsPerElm_d = create_mirror_view_and_copy(mpsPerElm, numElms); auto active_mp2Elm_d = create_mirror_view_and_copy(active_mp2Elm.data(), numActiveMPs); auto active_mpIDs_d = create_mirror_view_and_copy(active_mpIDs.data(), numActiveMPs); - + delete ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; ((polyMPO::MPMesh*)p_mpmesh)->p_MPs = new polyMPO::MaterialPoints(numElms, numActiveMPs, mpsPerElm_d, active_mp2Elm_d, active_mpIDs_d, elm2global); @@ -198,7 +197,7 @@ void polympo_startRebuildMPs_f(MPMesh_ptr p_mpmesh, } } -void polympo_startRebuildMPs_f2(MPMesh_ptr p_mpmesh, +void polympo_startRebuildMPs2_f(MPMesh_ptr p_mpmesh, const int sizeMP2elm, const int* elem_ids, const int nMPs_delete, @@ -235,7 +234,7 @@ void polympo_startRebuildMPs_f2(MPMesh_ptr p_mpmesh, int numDeletedMPs = pumipic::getLastValue(numDeletedMPs_d); assert(nMPs_delete==numDeletedMPs); p_MPs->startRebuild(mp2Elm, nMPs_add, recvMPs_elm_d, recvMPs_ids_d); - pumipic::RecordTime("polympo_startRebuildMPs_f2", timer.seconds()); + pumipic::RecordTime("polympo_startRebuildMPs2_f", timer.seconds()); } @@ -297,7 +296,7 @@ void polympo_getMPCurElmID_f(MPMesh_ptr p_mpmesh, auto getElmId = PS_LAMBDA(const int&, const int& mp, const int& mask){ if(mask){ - mpCurElmIDCopy(mpAppID(mp)) = mpCurElmID(mp)+elmIDoffset; + mpCurElmIDCopy(mpAppID(mp)) = mpCurElmID(mp)+elmIDoffset; } }; p_MPs->parallel_for(getElmId, "get mpCurElmID"); @@ -380,7 +379,6 @@ void polympo_setMPTgtPositions_f(MPMesh_ptr p_mpmesh, auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; PMT_ALWAYS_ASSERT(nComps == vec3d_nEntries); PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - //printf("NumMPs %d %d \n", numMPs, p_MPs->getCount()); //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); kkViewHostU mpPositionsIn_h(mpPositionsIn,nComps,numMPs); @@ -950,9 +948,6 @@ void polympo_setMeshDualTriangleArea_f(MPMesh_ptr p_mpmesh, const int nVertices, checkMPMeshValid(p_mpmesh); auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); - //copy the host array to the device auto dualArea = p_mesh->getMeshField(); auto h_dualArea = Kokkos::create_mirror_view(dualArea); @@ -968,9 +963,6 @@ void polympo_getMeshDualTriangleArea_f(MPMesh_ptr p_mpmesh, const int nVertices, checkMPMeshValid(p_mpmesh); auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); - //copy the device to host auto dualArea = p_mesh->getMeshField(); auto h_dualArea = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), dualArea); @@ -1183,28 +1175,28 @@ void polympo_getMeshVtxOnSurfDispIncr_f(MPMesh_ptr p_mpmesh, const int nComps, c bool polympo_push1P_f(MPMesh_ptr p_mpmesh){ checkMPMeshValid(p_mpmesh); - bool is_migrating=((polyMPO::MPMesh*)p_mpmesh) ->push1P(); + bool is_migrating=((polyMPO::MPMesh*)p_mpmesh)->push1P(); return is_migrating; } void polympo_push_ahead_f(MPMesh_ptr p_mpmesh){ checkMPMeshValid(p_mpmesh); - ((polyMPO::MPMesh*)p_mpmesh) ->push_ahead(); + ((polyMPO::MPMesh*)p_mpmesh)->push_ahead(); } void polympo_push_swap_f(MPMesh_ptr p_mpmesh){ checkMPMeshValid(p_mpmesh); - ((polyMPO::MPMesh*)p_mpmesh) ->push_swap(); + ((polyMPO::MPMesh*)p_mpmesh)->push_swap(); } void polympo_push_swap_pos_f(MPMesh_ptr p_mpmesh){ checkMPMeshValid(p_mpmesh); - ((polyMPO::MPMesh*)p_mpmesh) ->push_swap_pos(); + ((polyMPO::MPMesh*)p_mpmesh)->push_swap_pos(); } void polympo_push_f(MPMesh_ptr p_mpmesh){ checkMPMeshValid(p_mpmesh); - ((polyMPO::MPMesh*)p_mpmesh) ->push(); + ((polyMPO::MPMesh*)p_mpmesh)->push(); } //TODO skeleton of reconstruction functions @@ -1248,21 +1240,34 @@ void polympo_setReconstructionOfStress_f(MPMesh_ptr p_mpmesh, const int order, c //With MPI communication done via MPAS -void polympo_vtxSubAssemblyIceArea_f(MPMesh_ptr p_mpmesh, int size1, int size2, double* array, int comp){ +void polympo_vtxSubAssemblyIceArea_f(MPMesh_ptr p_mpmesh, int size1, int size2, int comp, double* array){ checkMPMeshValid(p_mpmesh); auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); - mpmesh->subAssemblyVtx1(size1, size2, array, comp); + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + PMT_ALWAYS_ASSERT(size1 <= maxVtxsPerElm); + PMT_ALWAYS_ASSERT(size2 == p_mesh->getNumElements()+1); + PMT_ALWAYS_ASSERT(comp == 0 || comp== 1); + mpmesh->subAssemblyVtx1(size1, size2, comp, array); } -void polympo_vtxSubAssemblyVelocity_f(MPMesh_ptr p_mpmesh, int size1, int size2, double* array, int comp){ + +void polympo_vtxSubAssemblyVelocity_f(MPMesh_ptr p_mpmesh, int size1, int size2, int comp, double* array){ checkMPMeshValid(p_mpmesh); auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); - mpmesh->subAssemblyVtx1(size1, size2, array, comp); + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + PMT_ALWAYS_ASSERT(size1 <= maxVtxsPerElm); + PMT_ALWAYS_ASSERT(size2 == p_mesh->getNumElements()+1); + PMT_ALWAYS_ASSERT(comp == 0 || comp== 1); + mpmesh->subAssemblyVtx1(size1, size2, comp, array); } + void polympo_subAssemblyCoeffs_f(MPMesh_ptr p_mpmesh, int dim1, int dim2, double* m11, double* m12, double* m13, double* m14, double* m22, double* m23, double* m24, double* m33, double* m34, double* m44){ checkMPMeshValid(p_mpmesh); + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + PMT_ALWAYS_ASSERT(dim1 <= maxVtxsPerElm); + PMT_ALWAYS_ASSERT(dim2 == p_mesh->getNumElements()+1); auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); mpmesh->subAssemblyCoeffs(dim1, dim2, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44); } @@ -1321,8 +1326,6 @@ void polympo_setVtxGlobal_f(MPMesh_ptr p_mpmesh, const int nVertices, const int* for (int i = 0; i < nVertices; i++) { arrayHost(i) = array[i] - 1; // TODO right now elmID offset is set after MPs initialized } - //check the size - //PMT_ALWAYS_ASSERT(nVertices == p_mesh->getNumVertices()); Kokkos::View vtxGlobal("vtxGlobal",nVertices); Kokkos::deep_copy(vtxGlobal, arrayHost); @@ -1331,7 +1334,7 @@ void polympo_setVtxGlobal_f(MPMesh_ptr p_mpmesh, const int nVertices, const int* int polympo_getMPCount_f(MPMesh_ptr p_mpmesh) { auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - return p_MPs->getCount(); // This returns the int you're after + return p_MPs->getCount(); } void polympo_enableTiming_f(){ diff --git a/src/pmpo_c.h b/src/pmpo_c.h index 6d9eda9..c05cbf0 100644 --- a/src/pmpo_c.h +++ b/src/pmpo_c.h @@ -21,7 +21,7 @@ void polympo_setMPICommunicator_f(MPMesh_ptr p_mpmesh, MPI_Fint fcomm); //MP info void polympo_createMPs_f(MPMesh_ptr p_mpmesh, const int numElms, const int numMPs, int* mpsPerElm, const int* mp2Elm, const int* isMPActive); void polympo_startRebuildMPs_f(MPMesh_ptr p_mpmesh, const int numMPs, const int* allTgtMpElmIn, const int* addedMPMask); -void polympo_startRebuildMPs_f2(MPMesh_ptr p_mpmesh, const int size1, const int* arg1, const int size2, const int size3, int* arg2, int* arg3); +void polympo_startRebuildMPs2_f(MPMesh_ptr p_mpmesh, const int size1, const int* arg1, const int size2, const int size3, int* arg2, int* arg3); int polympo_getMPCount_f(MPMesh_ptr p_mpmesh); @@ -113,8 +113,8 @@ void polympo_setReconstructionOfStress_f(MPMesh_ptr p_mpmesh, const int order, c void polympo_applyReconstruction_f(MPMesh_ptr p_mpmesh); //Reconstruction using MPAS -void polympo_vtxSubAssemblyIceArea_f(MPMesh_ptr p_mpmesh, int size1, int size2, double* array, int comp); -void polympo_vtxSubAssemblyVelocity_f(MPMesh_ptr p_mpmesh, int size1, int size2, double* array, int comp); +void polympo_vtxSubAssemblyIceArea_f(MPMesh_ptr p_mpmesh, int size1, int size2, int comp, double* array); +void polympo_vtxSubAssemblyVelocity_f(MPMesh_ptr p_mpmesh, int size1, int size2, int comp, double* array); void polympo_subAssemblyCoeffs_f(MPMesh_ptr p_mpmesh, int dim1, int dim2, double* m11, double* m12, double* m13, double* m14, double* m22, double* m23, double* m24, double* m33, double* m34, diff --git a/src/pmpo_fortran.f90 b/src/pmpo_fortran.f90 index 7afb646..22a7635 100644 --- a/src/pmpo_fortran.f90 +++ b/src/pmpo_fortran.f90 @@ -90,7 +90,7 @@ subroutine polympo_startRebuildMPs(mpMesh, numMPs, allMP2Elm, addedMPMask) & subroutine polympo_startRebuildMPs2(mpMesh, size1, arg1, size2, size3, arg2, arg3) & - bind(C, NAME='polympo_startRebuildMPs_f2') + bind(C, NAME='polympo_startRebuildMPs2_f') use :: iso_c_binding type(c_ptr), value :: mpMesh integer(c_int), value :: size1 @@ -910,14 +910,15 @@ subroutine polympo_setReconstructionOfStress(mpMesh, order, meshEntType) & end subroutine - subroutine polympo_vtxSubAssemblyIceArea(mpMesh, size1, size2, array, comp) & + subroutine polympo_vtxSubAssemblyIceArea(mpMesh, size1, size2, comp, array) & bind(C, NAME='polympo_vtxSubAssemblyIceArea_f') use :: iso_c_binding type(c_ptr), value :: mpMesh integer(c_int), value :: size1, size2, comp type(c_ptr), value :: array end subroutine - subroutine polympo_vtxSubAssemblyVelocity(mpMesh, size1, size2, array, comp) & + + subroutine polympo_vtxSubAssemblyVelocity(mpMesh, size1, size2, comp, array) & bind(C, NAME='polympo_vtxSubAssemblyVelocity_f') use :: iso_c_binding type(c_ptr), value :: mpMesh diff --git a/src/pmpo_materialPoints.cpp b/src/pmpo_materialPoints.cpp index 46fb41a..4037f99 100644 --- a/src/pmpo_materialPoints.cpp +++ b/src/pmpo_materialPoints.cpp @@ -114,7 +114,7 @@ void MaterialPoints::finishRebuild() { updateMaxAppID(); ps::destroyViews(rebuildFields.addedSlices_h); ps::destroyViews(addedSlices_d); - rebuildFields.ongoing = false; + rebuildFields.ongoing = false; } MPI_Comm MaterialPoints::getMPIComm() { diff --git a/src/pmpo_materialPoints.hpp b/src/pmpo_materialPoints.hpp index 12ff130..f8391d1 100644 --- a/src/pmpo_materialPoints.hpp +++ b/src/pmpo_materialPoints.hpp @@ -20,8 +20,6 @@ using defaultSpace = Kokkos::DefaultExecutionSpace::memory_space; typedef std::function IntFunc; typedef std::function IntIntFunc; - - enum MaterialPointSlice { MPF_Status = 0, MPF_Cur_Elm_ID, @@ -241,7 +239,7 @@ class MaterialPoints { auto xyz_geo = grid_rotation_backward(xyz_rot); lat_lon_from_xyz(geoLat, geoLon, xyz_geo, radius); } - + // x=cosLon cosLat, y=sinLon cosLat, z= sinLat tgtPosXYZ(mp,0) = radius * std::cos(geoLon) * std::cos(geoLat); tgtPosXYZ(mp,1) = radius * std::sin(geoLon) * std::cos(geoLat); tgtPosXYZ(mp,2) = radius * std::sin(geoLat); diff --git a/src/pmpo_utils.hpp b/src/pmpo_utils.hpp index 7a2fdcd..b9221b9 100644 --- a/src/pmpo_utils.hpp +++ b/src/pmpo_utils.hpp @@ -304,12 +304,12 @@ void CholeskySolve4d_UnitRHS(Matrix4d& A, double* x){ double a_00=A(0,0); if (A(0,0)==0){ x[0]=0; - x[1]=0; - x[2]=0; - x[3]=0; + x[1]=0; + x[2]=0; + x[3]=0; return; } - + A(0,0) = std::sqrt(A(0,0)); A(0,1) /= A(0,0); A(0,2) /= A(0,0); @@ -317,48 +317,48 @@ void CholeskySolve4d_UnitRHS(Matrix4d& A, double* x){ double diag = A(1,1) - A(0,1)*A(0,1); if(diag>EPSILON){ - A(1,1)=std::sqrt(diag); - A(1,2)=(A(1,2)-A(0,1)*A(0,2))/A(1,1); - A(1,3)=(A(1,3)-A(0,1)*A(0,3))/A(1,1); + A(1,1)=std::sqrt(diag); + A(1,2)=(A(1,2)-A(0,1)*A(0,2))/A(1,1); + A(1,3)=(A(1,3)-A(0,1)*A(0,3))/A(1,1); } else{ A(1,1)=0.0; - A(1,2)=0.0; - A(1,3)=0.0; + A(1,2)=0.0; + A(1,3)=0.0; } - + diag = A(2,2) - A(0,2)*A(0,2)-A(1,2)*A(1,2); if(diag>EPSILON){ A(2,2)=std::sqrt(diag); A(2,3)=(A(2,3)-A(0,2)*A(0,3)-A(1,2)*A(1,3))/A(2,2); } else{ - A(2,2)=0.0; - A(2,3)=0.0; + A(2,2)=0.0; + A(2,3)=0.0; } diag=A(3,3)-A(0,3)*A(0,3)-A(1,3)*A(1,3)-A(2,3)*A(2,3); if(diag>EPSILON) - A(3,3)=std::sqrt(diag); + A(3,3)=std::sqrt(diag); else - A(3,3)=0.0; + A(3,3)=0.0; if(abs(A(1,1)) Date: Tue, 12 Aug 2025 17:53:15 -0400 Subject: [PATCH 36/42] RP changes1_v1 --- src/pmpo_MPMesh.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index 5a1f9b5..200189a 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -156,7 +156,7 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ int elmID = elm2ElmConn(iElm,i)-1; //New delta - Vec3d center(elmCenter(elmID, 0), elmCenter(elmID, 1), elmCenter(elmID, 2)); + Vec3d center(elmCenter(elmID, 0), elmCenter(elmID, 1), elmCenter(elmID, 2)); delta = MPnew - center; double neighborDistSq = delta[0]*delta[0] + delta[1]*delta[1] + delta[2]*delta[2]; @@ -165,6 +165,7 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ minDistSq = neighborDistSq; } } + if(closestElm<0){ MPs2Elm(mp) = iElm; if (elm2Process.size() > 0) @@ -206,6 +207,7 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ Kokkos::deep_copy(h_resultRight, resultRight); Kokkos::deep_copy(h_mpTgtPos, mpTgtPosArray); // printVTP file + char* fileOutput = (char *)malloc(sizeof(char) * 256); sprintf(fileOutput, "polyMPOCVTTrackingElmCenter_MPtracks_%d.vtp", printVTPIndex); FILE * pFile = fopen(fileOutput,"w"); @@ -229,7 +231,6 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ fprintf(pFile," \n \n \n \n\n"); fclose(pFile); } - pumipic::RecordTime("PolyMPO_CVTTrackingElmCenterBased", timer.seconds()); } From 7f362c0e7cba3cc30355823df85cc3fbc8d89800 Mon Sep 17 00:00:00 2001 From: Nath Date: Tue, 12 Aug 2025 17:57:27 -0400 Subject: [PATCH 37/42] RP changes1_v2 --- src/pmpo_MPMesh.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index 200189a..7984fea 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -206,8 +206,8 @@ void MPMesh::CVTTrackingElmCenterBased(const int printVTPIndex){ Kokkos::deep_copy(h_resultLeft, resultLeft); Kokkos::deep_copy(h_resultRight, resultRight); Kokkos::deep_copy(h_mpTgtPos, mpTgtPosArray); - // printVTP file + // printVTP file char* fileOutput = (char *)malloc(sizeof(char) * 256); sprintf(fileOutput, "polyMPOCVTTrackingElmCenter_MPtracks_%d.vtp", printVTPIndex); FILE * pFile = fopen(fileOutput,"w"); From 3545660219c1b2e851003fdea7f2f1ee99866001 Mon Sep 17 00:00:00 2001 From: Nath Date: Tue, 12 Aug 2025 17:59:54 -0400 Subject: [PATCH 38/42] RP changes1_v3 --- src/pmpo_MPMesh.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index 7984fea..73c8500 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -2,6 +2,7 @@ #include "pmpo_utils.hpp" #include "pmpo_MPMesh.hpp" #include "pmpo_wachspressBasis.hpp" + namespace polyMPO{ void printVTP_mesh(MPMesh& mpMesh, int printVTPIndex=-1); From 775ca49dedfe7fffdf232a657f1d18bca1f74246 Mon Sep 17 00:00:00 2001 From: Nath Date: Wed, 13 Aug 2025 18:47:04 -0400 Subject: [PATCH 39/42] PR changes2_v0 --- "\\" | 1350 ++++++++++++++++++++++++++++++++++ src/pmpo_MPMesh.cpp | 2 +- src/pmpo_MPMesh.hpp | 18 +- src/pmpo_MPMesh_assembly.hpp | 100 +-- src/pmpo_c.cpp | 59 +- src/pmpo_c.h | 14 +- src/pmpo_defines.h | 2 +- src/pmpo_fortran.f90 | 18 +- src/pmpo_materialPoints.cpp | 2 +- src/pmpo_utils.hpp | 241 +++--- src/pmpo_wachspressBasis.hpp | 4 +- 11 files changed, 1579 insertions(+), 231 deletions(-) create mode 100644 "\\" diff --git "a/\\" "b/\\" new file mode 100644 index 0000000..f315b4a --- /dev/null +++ "b/\\" @@ -0,0 +1,1350 @@ +#include "pmpo_createTestMPMesh.hpp" +#include "pmpo_defines.h" +#include "pmpo_c.h" +#include "pmpo_MPMesh_assembly.hpp" +#include + +namespace{ + std::vector p_mpmeshes;////store the p_mpmeshes that is legal + + void checkMPMeshValid(MPMesh_ptr p_mpmesh){ + auto p_mpmeshIter = std::find(p_mpmeshes.begin(),p_mpmeshes.end(),p_mpmesh); + PMT_ALWAYS_ASSERT(p_mpmeshIter != p_mpmeshes.end()); + } +} + +void polympo_initialize_f() { + int isMPIInit; + MPI_Initialized(&isMPIInit); + PMT_ALWAYS_ASSERT(isMPIInit); + Kokkos::initialize(); +} + +void polympo_finalize_f() { + Kokkos::finalize(); +} + +MPMesh_ptr polympo_createMPMesh_f(const int testMeshOption, const int testMPOption) { + polyMPO::Mesh* p_mesh; + if(testMeshOption){ + int replicateFactor = 1; + p_mesh = polyMPO::initTestMesh(testMeshOption, replicateFactor); + }else{ + p_mesh = new polyMPO::Mesh(); + } + polyMPO::MaterialPoints* p_mps; + if(testMPOption){ + PMT_ALWAYS_ASSERT(testMeshOption >= 1); + p_mps = polyMPO::initTestMPs(p_mesh, testMPOption); + }else{ + p_mps = new polyMPO::MaterialPoints(); + } + MPMesh_ptr p_mpMeshReturn = (MPMesh_ptr) new polyMPO::MPMesh(p_mesh, p_mps); + p_mpmeshes.push_back(p_mpMeshReturn); + return p_mpMeshReturn; +} + +void polympo_deleteMPMesh_f(MPMesh_ptr p_mpmesh) { + //check mpMesh is valid + auto p_mpmeshIter = std::find(p_mpmeshes.begin(),p_mpmeshes.end(),p_mpmesh); + PMT_ALWAYS_ASSERT(p_mpmeshIter != p_mpmeshes.end()); + p_mpmeshes.erase(p_mpmeshIter); + delete (polyMPO::MPMesh*)p_mpmesh; +} + +void polympo_setMPICommunicator_f(MPMesh_ptr p_mpmesh, MPI_Fint fcomm){ + checkMPMeshValid(p_mpmesh); + MPI_Comm comm = MPI_Comm_f2c(fcomm); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + p_MPs->setMPIComm(comm); +} + +void polympo_createMPs_f(MPMesh_ptr p_mpmesh, + const int numElms, + const int numMPs, // total number of MPs which is >= no of active MPs + int* mpsPerElm, + const int* mp2Elm, + const int* isMPActive) { + checkMPMeshValid(p_mpmesh); + //the mesh must be fixed/set before adding MPs + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + PMT_ALWAYS_ASSERT(!p_mesh->meshEditable()); + PMT_ALWAYS_ASSERT(p_mesh->getNumElements() == numElms); + + //Find the total no of MPs across all ranks + //And loop over all MPs and find the smallest element id associated across a MP + int numActiveMPs = 0; + int minElmID = INT_MAX; + for(int i = 0; i < numMPs; i++) { + if(isMPActive[i] == MP_ACTIVE) { + numActiveMPs++; + if(mp2Elm[i] < minElmID) + minElmID = mp2Elm[i]; + } + } + int globalNumActiveMPs = 0; + int globalMinElmID; + MPI_Allreduce(&numActiveMPs, &globalNumActiveMPs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + MPI_Allreduce(&minElmID, &globalMinElmID, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD); + PMT_ALWAYS_ASSERT(globalNumActiveMPs>0); + + //Loop over all mesh elements 0,1,... and find the first element that has an associated MP + int firstElmWithMPs=INT_MAX; + for (int i=0; i active_mpIDs(numMPs); + std::vector active_mp2Elm(numMPs); + numActiveMPs = 0; + for(int i=0; igetElmGlobal(); + auto mpsPerElm_d = create_mirror_view_and_copy(mpsPerElm, numElms); + auto active_mp2Elm_d = create_mirror_view_and_copy(active_mp2Elm.data(), numActiveMPs); + auto active_mpIDs_d = create_mirror_view_and_copy(active_mpIDs.data(), numActiveMPs); + + delete ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + ((polyMPO::MPMesh*)p_mpmesh)->p_MPs = + new polyMPO::MaterialPoints(numElms, numActiveMPs, mpsPerElm_d, active_mp2Elm_d, active_mpIDs_d, elm2global); + + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + p_MPs->setElmIDoffset(offset); + +} + +void polympo_startRebuildMPs_f(MPMesh_ptr p_mpmesh, + const int numMPs, // Total number of MPs which is GREATER than or equal to number of active MPs + const int* allMP2Elm, + const int* addedMPMask) { + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); + PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + + int offset = p_MPs->getElmIDoffset(); + std::vector added_mpIDs(numMPs); + std::vector added_mp2Elm(numMPs); + int numAddedMPs = 0; + for(int i=0; igetCapacity(); // pumipic expects full capacity to rebuild + Kokkos::View mp2Elm("mp2Elm", internalMPCapacity); + auto mpAppID = p_MPs->getData(); + + auto added_mp2Elm_d = create_mirror_view_and_copy(added_mp2Elm.data(), numAddedMPs); + auto added_mpIDs_d = create_mirror_view_and_copy(added_mpIDs.data(), numAddedMPs); + auto addedMPMask_d = create_mirror_view_and_copy(addedMPMask, numMPs); + auto mpMP2ElmIn_d = create_mirror_view_and_copy(allMP2Elm, numMPs); + + Kokkos::View numDeletedMPs_d("numDeletedMPs", 1); + auto setMP2Elm = PS_LAMBDA(const int&, const int& mp, const int& mask) { + if(mask) { + if (addedMPMask_d[mpAppID(mp)] == MP_ACTIVE) //two MPs can not occupy the same slot + mp2Elm(mp) = MP_DELETE; + else + mp2Elm(mp) = mpMP2ElmIn_d(mpAppID(mp)); + if (mp2Elm(mp) == MP_DELETE) + Kokkos::atomic_increment(&numDeletedMPs_d(0)); + } + }; + p_MPs->parallel_for(setMP2Elm, "setMP2Elm"); + + int numDeletedMPs = pumipic::getLastValue(numDeletedMPs_d); + PMT_ALWAYS_ASSERT(numAddedMPs > 0 || numDeletedMPs > 0); + + p_MPs->startRebuild(mp2Elm, numAddedMPs, added_mp2Elm_d, added_mpIDs_d, addedMPMask_d); + + // check mpAppID is unique (on GPUs) + if (p_MPs->getOpMode() == polyMPO::MP_DEBUG){ + mpAppID = p_MPs->getData(); + Kokkos::View mpAppIDCount("mpAppIDCount", p_MPs->getCount()); + auto checkAppIDs = PS_LAMBDA(const int&, const int& mp, const int& mask){ + if(mask) { + int prev = Kokkos::atomic_fetch_add(&mpAppIDCount(mpAppID(mp)), 1); + assert(prev == 0); + } + }; + p_MPs->parallel_for(checkAppIDs, "checkAppIDs"); + } +} + +void polympo_startRebuildMPs2_f(MPMesh_ptr p_mpmesh, + const int sizeMP2elm, + const int* elem_ids, + const int nMPs_delete, + const int nMPs_add, + int* recvMPs_elm, + int* recvMPs_ids) { + + Kokkos::Timer timer; + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + int offset = p_MPs->getElmIDoffset(); + + for (int k=0; k mp2Elm("mp2Elm", p_MPs->getCapacity()); + Kokkos::View numDeletedMPs_d("numDeletedMPs", 1); + auto mpAppID = p_MPs->getData(); + auto setMP2Elm = PS_LAMBDA(const int& e, const int& mp, const int& mask) { + if(mask) { + mp2Elm(mp) = elem_ids_d(mpAppID(mp))-offset; + if (mp2Elm(mp) == MP_DELETE){ + Kokkos::atomic_increment(&numDeletedMPs_d(0)); + } + } + }; + p_MPs->parallel_for(setMP2Elm, "setMP2Elm"); + int numDeletedMPs = pumipic::getLastValue(numDeletedMPs_d); + assert(nMPs_delete==numDeletedMPs); + p_MPs->startRebuild(mp2Elm, nMPs_add, recvMPs_elm_d, recvMPs_ids_d); + pumipic::RecordTime("polympo_startRebuildMPs2_f", timer.seconds()); +} + +void polympo_finishRebuildMPs_f(MPMesh_ptr p_mpmesh){ + Kokkos::Timer timer; + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + p_MPs->finishRebuild(); + pumipic::RecordTime("polympo_finishRebuildMPs_f", timer.seconds()); +} + +void polympo_setAppIDFunc_f(MPMesh_ptr p_mpmesh, IntVoidFunc getNext, void* appIDs) { + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + polyMPO::IntFunc getNextAppID = [getNext, appIDs]() { return getNext(appIDs); }; + p_MPs->setAppIDFunc(getNextAppID); +} + +void polympo_getMPTgtElmID_f(MPMesh_ptr p_mpmesh, + const int numMPs, + int* elmIDs){ + Kokkos::Timer timer; + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); + //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + auto mpTgtElmID = p_MPs->getData(); + auto mpAppID = p_MPs->getData(); + auto elmIDoffset = p_MPs->getElmIDoffset(); + + kkIntViewHostU arrayHost(elmIDs,numMPs); + polyMPO::IntView mpTgtElmIDCopy("mpTgtElmIDNewValue",numMPs); + + auto setTgtElmId = PS_LAMBDA(const int&, const int& mp, const int& mask){ + if(mask){ + mpTgtElmIDCopy(mpAppID(mp)) = mpTgtElmID(mp)+elmIDoffset; + } + }; + p_MPs->parallel_for(setTgtElmId, "set mpTgtElmID"); + Kokkos::deep_copy( arrayHost, mpTgtElmIDCopy); + pumipic::RecordTime("PolyMPO_getMPTgtElmID", timer.seconds()); +} + +void polympo_getMPCurElmID_f(MPMesh_ptr p_mpmesh, + const int numMPs, + int* elmIDs){ + + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); + //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + auto mpCurElmID = p_MPs->getData(); + auto mpAppID = p_MPs->getData(); + auto elmIDoffset = p_MPs->getElmIDoffset(); + + kkIntViewHostU arrayHost(elmIDs,numMPs); + polyMPO::IntView mpCurElmIDCopy("mpCurElmIDNewValue",numMPs); + + auto getElmId = PS_LAMBDA(const int&, const int& mp, const int& mask){ + if(mask){ + mpCurElmIDCopy(mpAppID(mp)) = mpCurElmID(mp)+elmIDoffset; + } + }; + p_MPs->parallel_for(getElmId, "get mpCurElmID"); + Kokkos::deep_copy( arrayHost, mpCurElmIDCopy); + +} + +void polympo_setMPLatLonRotatedFlag_f(MPMesh_ptr p_mpmesh, const int isRotateFlag){ + //chech validity + checkMPMeshValid(p_mpmesh); + ((polyMPO::MPMesh*)p_mpmesh)->p_MPs->setRotatedFlag(isRotateFlag>0); + +} + +void polympo_setMPPositions_f(MPMesh_ptr p_mpmesh, + const int nComps, + const int numMPs, + const double* mpPositionsIn){ + Kokkos::Timer timer; + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + PMT_ALWAYS_ASSERT(nComps == vec3d_nEntries); + PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); + //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + kkViewHostU mpPositionsIn_h(mpPositionsIn,nComps,numMPs); + + if (p_MPs->rebuildOngoing()) { + p_MPs->setRebuildMPSlice(mpPositionsIn_h); + return; + } + + auto mpPositions = p_MPs->getData(); + auto mpAppID = p_MPs->getData(); + + Kokkos::View mpPositionsIn_d("mpPositionsDevice",vec3d_nEntries,numMPs); + Kokkos::deep_copy(mpPositionsIn_d, mpPositionsIn_h); + auto setPos = PS_LAMBDA(const int&, const int& mp, const int& mask){ + if(mask){ + mpPositions(mp,0) = mpPositionsIn_d(0, mpAppID(mp)); + mpPositions(mp,1) = mpPositionsIn_d(1, mpAppID(mp)); + mpPositions(mp,2) = mpPositionsIn_d(2, mpAppID(mp)); + } + }; + p_MPs->parallel_for(setPos, "setMPPositions"); + pumipic::RecordTime("PolyMPO_setMPPositions", timer.seconds()); +} + +void polympo_getMPPositions_f(MPMesh_ptr p_mpmesh, + const int nComps, + const int numMPs, + double* mpPositionsHost){ + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + PMT_ALWAYS_ASSERT(nComps == vec3d_nEntries); + PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); + //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + + auto mpPositions = p_MPs->getData(); + auto mpAppID = p_MPs->getData(); + Kokkos::View mpPositionsCopy("mpPositionsCopy",vec3d_nEntries,numMPs); + auto getPos = PS_LAMBDA(const int&, const int& mp, const int& mask){ + if(mask){ + mpPositionsCopy(0,mpAppID(mp)) = mpPositions(mp,0); + mpPositionsCopy(1,mpAppID(mp)) = mpPositions(mp,1); + mpPositionsCopy(2,mpAppID(mp)) = mpPositions(mp,2); + } + }; + p_MPs->parallel_for(getPos, "getMPPositions"); + kkDbl2dViewHostU arrayHost(mpPositionsHost,nComps,numMPs); + Kokkos::deep_copy(arrayHost, mpPositionsCopy); +} + + +void polympo_setMPTgtPositions_f(MPMesh_ptr p_mpmesh, + const int nComps, + const int numMPs, + const double* mpPositionsIn){ + Kokkos::Timer timer; + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + PMT_ALWAYS_ASSERT(nComps == vec3d_nEntries); + PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); + //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + kkViewHostU mpPositionsIn_h(mpPositionsIn,nComps,numMPs); + + if (p_MPs->rebuildOngoing()) { + p_MPs->setRebuildMPSlice(mpPositionsIn_h); + return; + } + + auto mpPositions = p_MPs->getData(); + auto mpAppID = p_MPs->getData(); + Kokkos::View mpPositionsIn_d("mpPositionsDevice",vec3d_nEntries,numMPs); + Kokkos::deep_copy(mpPositionsIn_d, mpPositionsIn_h); + auto setPos = PS_LAMBDA(const int&, const int& mp, const int& mask){ + if(mask){ + mpPositions(mp,0) = mpPositionsIn_d(0, mpAppID(mp)); + mpPositions(mp,1) = mpPositionsIn_d(1, mpAppID(mp)); + mpPositions(mp,2) = mpPositionsIn_d(2, mpAppID(mp)); + } + }; + p_MPs->parallel_for(setPos, "setMPPositions"); + pumipic::RecordTime("PolyMPO_setMPTgtPositions", timer.seconds()); +} + +void polympo_getMPTgtPositions_f(MPMesh_ptr p_mpmesh, + const int nComps, + const int numMPs, + double* mpPositionsHost){ + Kokkos::Timer timer; + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + PMT_ALWAYS_ASSERT(nComps == vec3d_nEntries); + PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); + //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + + auto mpPositions = p_MPs->getData(); + auto mpAppID = p_MPs->getData(); + Kokkos::View mpPositionsCopy("mpPositionsCopy",vec3d_nEntries,numMPs); + auto getPos = PS_LAMBDA(const int&, const int& mp, const int& mask){ + if(mask){ + mpPositionsCopy(0,mpAppID(mp)) = mpPositions(mp,0); + mpPositionsCopy(1,mpAppID(mp)) = mpPositions(mp,1); + mpPositionsCopy(2,mpAppID(mp)) = mpPositions(mp,2); + } + }; + p_MPs->parallel_for(getPos, "getMPPositions"); + kkDbl2dViewHostU arrayHost(mpPositionsHost,nComps,numMPs); + Kokkos::deep_copy(arrayHost, mpPositionsCopy); + pumipic::RecordTime("PolyMPO_getMPTgtPositions", timer.seconds()); +} + + +void polympo_setMPRotLatLon_f(MPMesh_ptr p_mpmesh, + const int nComps, + const int numMPs, + const double* mpRotLatLonIn){ + Kokkos::Timer timer; + static int callCount = 0; + PMT_ALWAYS_ASSERT(callCount == 0); + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); + PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); + PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + + auto mpRotLatLon = p_MPs->getData(); + auto mpAppID = p_MPs->getData(); + kkViewHostU mpRotLatLonIn_h(mpRotLatLonIn,nComps,numMPs); + Kokkos::View mpRotLatLonIn_d("mpRotLatLonDevice",vec2d_nEntries,numMPs); + Kokkos::deep_copy(mpRotLatLonIn_d, mpRotLatLonIn_h); + auto setPos = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ + if(mask){ + mpRotLatLon(mp,0) = mpRotLatLonIn_d(0, mpAppID(mp)); + mpRotLatLon(mp,1) = mpRotLatLonIn_d(1, mpAppID(mp)); + } + }; + p_MPs->parallel_for(setPos, "setMPRotLatLon"); + callCount++; + pumipic::RecordTime("PolyMPO_setMPRotLatLon", timer.seconds()); +} + +void polympo_getMPRotLatLon_f(MPMesh_ptr p_mpmesh, + const int nComps, + const int numMPs, + double* mpRotLatLonHost){ + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); + PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); + PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + + auto mpRotLatLon = p_MPs->getData(); + auto mpAppID = p_MPs->getData(); + Kokkos::View mpRotLatLonCopy("mpRotLatLonCopy",vec2d_nEntries,numMPs); + auto getPos = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ + if(mask){ + mpRotLatLonCopy(0,mpAppID(mp)) = mpRotLatLon(mp,0); + mpRotLatLonCopy(1,mpAppID(mp)) = mpRotLatLon(mp,1); + } + }; + p_MPs->parallel_for(getPos, "getMPRotLatLon"); + kkDbl2dViewHostU arrayHost(mpRotLatLonHost,nComps,numMPs); + Kokkos::deep_copy(arrayHost, mpRotLatLonCopy); +} + + +void polympo_setMPTgtRotLatLon_f(MPMesh_ptr p_mpmesh, + const int nComps, + const int numMPs, + const double* mpRotLatLonIn){ + Kokkos::Timer timer; + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); + PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); + //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + + auto mpRotLatLon = p_MPs->getData(); + auto mpAppID = p_MPs->getData(); + kkViewHostU mpRotLatLonIn_h(mpRotLatLonIn,nComps,numMPs); + Kokkos::View mpRotLatLonIn_d("mpRotLatLonDevice",vec2d_nEntries,numMPs); + Kokkos::deep_copy(mpRotLatLonIn_d, mpRotLatLonIn_h); + auto setPos = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ + if(mask){ + mpRotLatLon(mp,0) = mpRotLatLonIn_d(0, mpAppID(mp)); + mpRotLatLon(mp,1) = mpRotLatLonIn_d(1, mpAppID(mp)); + } + }; + p_MPs->parallel_for(setPos, "setMPTgtRotLatLon"); + pumipic::RecordTime("PolyMPO_setMPTgtRotLatLon", timer.seconds()); +} + +void polympo_getMPTgtRotLatLon_f(MPMesh_ptr p_mpmesh, + const int nComps, + const int numMPs, + double* mpRotLatLonHost){ + Kokkos::Timer timer; + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); + PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); + //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + + auto mpRotLatLon = p_MPs->getData(); + auto mpAppID = p_MPs->getData(); + Kokkos::View mpRotLatLonCopy("mpRotLatLonCopy",vec2d_nEntries,numMPs); + auto getPos = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ + if(mask){ + mpRotLatLonCopy(0,mpAppID(mp)) = mpRotLatLon(mp,0); + mpRotLatLonCopy(1,mpAppID(mp)) = mpRotLatLon(mp,1); + } + }; + p_MPs->parallel_for(getPos, "getMPRotLatLon"); + kkDbl2dViewHostU arrayHost(mpRotLatLonHost,nComps,numMPs); + Kokkos::deep_copy(arrayHost, mpRotLatLonCopy); + pumipic::RecordTime("PolyMPO_getMPTgtRotLatLon", timer.seconds()); +} + + +void polympo_setMPMass_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpMassIn) { + Kokkos::Timer timer; + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + PMT_ALWAYS_ASSERT(nComps == 1); //TODO mp_sclr_t + PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); + //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + + auto mpMass = p_MPs->getData(); + auto mpAppID = p_MPs->getData(); + kkViewHostU mpMassIn_h(mpMassIn,nComps,numMPs); + Kokkos::View mpMassIn_d("mpMassDevice",nComps,numMPs); + Kokkos::deep_copy(mpMassIn_d, mpMassIn_h); + auto setMPMass = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ + if(mask){ + mpMass(mp,0) = mpMassIn_d(0, mpAppID(mp)); + } + }; + p_MPs->parallel_for(setMPMass, "setMPMass"); + pumipic::RecordTime("PolyMPO_setMPMass", timer.seconds()); +} + +void polympo_getMPMass_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, double* mpMassHost) { + Kokkos::Timer timer; + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + PMT_ALWAYS_ASSERT(nComps == 1); //TODO mp_sclr_t + PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); + //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + + auto mpMass = p_MPs->getData(); + auto mpAppID = p_MPs->getData(); + Kokkos::View mpMassCopy("mpMassCopy",nComps,numMPs); + auto getMPMass = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ + if(mask){ + mpMassCopy(0,mpAppID(mp)) = mpMass(mp,0); + } + }; + p_MPs->parallel_for(getMPMass, "getMPMass"); + kkDbl2dViewHostU arrayHost(mpMassHost,nComps,numMPs); + Kokkos::deep_copy(arrayHost, mpMassCopy); + pumipic::RecordTime("PolyMPO_getMPMass", timer.seconds()); +} + +void polympo_setMPVel_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpVelIn) { + Kokkos::Timer timer; + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); + PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); + //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + + auto mpVel = p_MPs->getData(); + auto mpAppID = p_MPs->getData(); + kkViewHostU mpVelIn_h(mpVelIn,nComps,numMPs); + Kokkos::View mpVelIn_d("mpVelDevice",vec2d_nEntries,numMPs); + Kokkos::deep_copy(mpVelIn_d, mpVelIn_h); + auto setMPVel = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ + if(mask){ + mpVel(mp,0) = mpVelIn_d(0, mpAppID(mp)); + mpVel(mp,1) = mpVelIn_d(1, mpAppID(mp)); + } + }; + p_MPs->parallel_for(setMPVel, "setMPVel"); + pumipic::RecordTime("PolyMPO_setMPVel", timer.seconds()); +} + +void polympo_getMPVel_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, double* mpVelHost) { + Kokkos::Timer timer; + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); + PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); + //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + + auto mpVel = p_MPs->getData(); + auto mpAppID = p_MPs->getData(); + Kokkos::View mpVelCopy("mpVelCopy",vec2d_nEntries,numMPs); + auto getMPVel = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ + if(mask){ + mpVelCopy(0,mpAppID(mp)) = mpVel(mp,0); + mpVelCopy(1,mpAppID(mp)) = mpVel(mp,1); + } + }; + p_MPs->parallel_for(getMPVel, "getMPVel"); + kkDbl2dViewHostU arrayHost(mpVelHost,nComps,numMPs); + Kokkos::deep_copy(arrayHost, mpVelCopy); + pumipic::RecordTime("PolyMPO_getMPVel", timer.seconds()); +} + +//TODO: implement these +void polympo_setMPStrainRate_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpStrainRateIn){ + checkMPMeshValid(p_mpmesh); + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + PMT_ALWAYS_ASSERT(nComps == 6); //TODO: mp_sym_mat3d_t + PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); + PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); + + auto mpStrainRate = p_MPs->getData(); + auto mpAppID = p_MPs->getData(); + kkViewHostU mpStrainRateIn_h(mpStrainRateIn,nComps,numMPs); + Kokkos::View mpStrainRateIn_d("mpStrainRateDevice",nComps,numMPs); + Kokkos::deep_copy(mpStrainRateIn_d, mpStrainRateIn_h); + auto setMPStrainRate = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ + if(mask){ + mpStrainRate(mp,0) = mpStrainRateIn_d(0, mpAppID(mp)); + mpStrainRate(mp,1) = mpStrainRateIn_d(1, mpAppID(mp)); + mpStrainRate(mp,2) = mpStrainRateIn_d(2, mpAppID(mp)); + mpStrainRate(mp,3) = mpStrainRateIn_d(3, mpAppID(mp)); + mpStrainRate(mp,4) = mpStrainRateIn_d(4, mpAppID(mp)); + mpStrainRate(mp,5) = mpStrainRateIn_d(5, mpAppID(mp)); + } + }; + p_MPs->parallel_for(setMPStrainRate, "setMPStrainRate"); +} +void polympo_getMPStrainRate_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, double* mpStrainRateHost){ + checkMPMeshValid(p_mpmesh); + std::cerr << "Error: This routine is not implemented yet\n"; + exit(1); + (void)p_mpmesh; + (void)nComps; + (void)numMPs; + (void)mpStrainRateHost; +} +void polympo_setMPStress_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpStressIn){ + checkMPMeshValid(p_mpmesh); + std::cerr << "Error: This routine is not implemented yet\n"; + exit(1); + (void)p_mpmesh; + (void)nComps; + (void)numMPs; + (void)mpStressIn; +} +void polympo_getMPStress_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, double* mpStressHost){ + checkMPMeshValid(p_mpmesh); + std::cerr << "Error: This routine is not implemented yet\n"; + exit(1); + (void)p_mpmesh; + (void)nComps; + (void)numMPs; + (void)mpStressHost; +} + +void polympo_startMeshFill_f(MPMesh_ptr p_mpmesh){ + checkMPMeshValid(p_mpmesh); + ((polyMPO::MPMesh*)p_mpmesh)->p_mesh->setMeshEdit(true); +} + +void polympo_endMeshFill_f(MPMesh_ptr p_mpmesh){ + checkMPMeshValid(p_mpmesh); + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + PMT_ALWAYS_ASSERT(p_mesh->meshEditable()); + p_mesh->setMeshEdit(false); +} + +void polympo_checkMeshMaxSettings_f(MPMesh_ptr p_mpmesh, const int maxEdges, const int vertexDegree){ + checkMPMeshValid(p_mpmesh); + PMT_ALWAYS_ASSERT(maxEdges <= maxVtxsPerElm); + PMT_ALWAYS_ASSERT(vertexDegree <= maxElmsPerVtx); +} + +void polympo_setMeshTypeGeneralPoly_f(MPMesh_ptr p_mpmesh){ + //chech validity + checkMPMeshValid(p_mpmesh); + ((polyMPO::MPMesh*)p_mpmesh)->p_mesh->setMeshType(polyMPO::mesh_general_polygonal); +} + +void polympo_setMeshTypeCVTPoly_f(MPMesh_ptr p_mpmesh){ + //chech validity + checkMPMeshValid(p_mpmesh); + ((polyMPO::MPMesh*)p_mpmesh)->p_mesh->setMeshType(polyMPO::mesh_CVT_polygonal); +} + +void polympo_setMeshGeomTypePlanar_f(MPMesh_ptr p_mpmesh){ + //chech validity + checkMPMeshValid(p_mpmesh); + ((polyMPO::MPMesh*)p_mpmesh)->p_mesh->setGeomType(polyMPO::geom_planar_surf); +} + +void polympo_setMeshGeomTypeSpherical_f(MPMesh_ptr p_mpmesh){ + //chech validity + checkMPMeshValid(p_mpmesh); + ((polyMPO::MPMesh*)p_mpmesh)->p_mesh->setGeomType(polyMPO::geom_spherical_surf); +} + +void polympo_setMeshSphereRadius_f(MPMesh_ptr p_mpmesh, const double sphereRadius){ + //chech validity + checkMPMeshValid(p_mpmesh); + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + PMT_ALWAYS_ASSERT(sphereRadius >= 0); + p_mesh->setSphereRadius(sphereRadius); +} + +void polympo_setMeshNumVtxs_f(MPMesh_ptr p_mpmesh, const int numVtxs){ + checkMPMeshValid(p_mpmesh); + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + p_mesh->setNumVtxs(numVtxs); + p_mesh->setMeshVtxBasedFieldSize(); +} + +void polympo_getMeshNumVtxs_f(MPMesh_ptr p_mpmesh, int & numVtxs) { + checkMPMeshValid(p_mpmesh); //chech vailidity + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + numVtxs = p_mesh->getNumVertices(); +} + +void polympo_setMeshNumElms_f(MPMesh_ptr p_mpmesh, const int numElms){ + checkMPMeshValid(p_mpmesh); + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + + auto elm2Vtx = polyMPO::IntVtx2ElmView("MeshElementsToVertices",numElms); + auto elm2Elm = polyMPO::IntElm2ElmView("MeshElementsToElements",numElms); + + p_mesh->setNumElms(numElms); + p_mesh->setElm2VtxConn(elm2Vtx); + p_mesh->setElm2ElmConn(elm2Elm); + p_mesh->setMeshElmBasedFieldSize(); +} + +void polympo_getMeshNumElms_f(MPMesh_ptr p_mpmesh, int & numElms) { + checkMPMeshValid(p_mpmesh); //chech vailidity + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + numElms = p_mesh->getNumElements(); +} + +void polympo_setMeshNumEdgesPerElm_f(MPMesh_ptr p_mpmesh, const int nCells, const int* array){ + //chech vailidity + checkMPMeshValid(p_mpmesh); + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + PMT_ALWAYS_ASSERT(p_mesh->meshEditable()); + + //check the size + PMT_ALWAYS_ASSERT(p_mesh->getNumElements()==nCells); + auto nEdgesPerElm = create_mirror_view_and_copy(array, nCells); + auto elm2VtxConn = p_mesh->getElm2VtxConn(); + auto elm2ElmConn = p_mesh->getElm2ElmConn(); + Kokkos::parallel_for("set nEdgesPerElm", nCells, KOKKOS_LAMBDA(const int elm){ + elm2VtxConn(elm,0) = nEdgesPerElm(elm); + elm2ElmConn(elm,0) = nEdgesPerElm(elm); + }); +} + +void polympo_setMeshElm2VtxConn_f(MPMesh_ptr p_mpmesh, const int maxEdges, const int nCells, const int* array){ + //chech vailidity + checkMPMeshValid(p_mpmesh); + kkViewHostU arrayHost(array,maxEdges,nCells); + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + PMT_ALWAYS_ASSERT(p_mesh->meshEditable()); + + //check the size + PMT_ALWAYS_ASSERT(maxEdges <= maxVtxsPerElm); + PMT_ALWAYS_ASSERT(nCells == p_mesh->getNumElements()); + + Kokkos::View elm2VtxArray("MeshElementsToVertices",maxEdges,nCells); + Kokkos::deep_copy(elm2VtxArray, arrayHost); + auto elm2VtxConn = p_mesh->getElm2VtxConn(); + Kokkos::parallel_for("set elm2VtxConn", nCells, KOKKOS_LAMBDA(const int elm){ + for(int i=0; i arrayHost(array,maxEdges,nCells); //Fortran is column-major + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + PMT_ALWAYS_ASSERT(p_mesh->meshEditable()); + + //check the size + PMT_ALWAYS_ASSERT(maxEdges <= maxVtxsPerElm); + PMT_ALWAYS_ASSERT(nCells == p_mesh->getNumElements()); + + Kokkos::View elm2ElmArray("MeshElementsToVertices",maxEdges,nCells); + Kokkos::deep_copy(elm2ElmArray, arrayHost); + auto elm2ElmConn = p_mesh->getElm2ElmConn(); + Kokkos::parallel_for("set elm2ElmConn", nCells, KOKKOS_LAMBDA(const int elm){ + for(int i=0; ip_mesh; + + //check the size + PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); + + //copy the host array to the device + auto coordsArray = p_mesh->getMeshField(); + auto h_coordsArray = Kokkos::create_mirror_view(coordsArray); + for(int i=0; ip_mesh; + + //check the size + PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); + + //copy the device to host + auto coordsArray = p_mesh->getMeshField(); + auto h_coordsArray = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), + coordsArray); + for(int i=0; ip_mesh; + + //check the size + PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); + + //copy the host array to the device + auto coordsArray = p_mesh->getMeshField(); + auto h_coordsArray = Kokkos::create_mirror_view(coordsArray); + for(int i=0; ip_mesh; + + //check the size + PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); + + //copy the device to host + auto coordsArray = p_mesh->getMeshField(); + auto h_coordsArray = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), + coordsArray); + for(int i=0; ip_mesh; + + //check the size + PMT_ALWAYS_ASSERT(p_mesh->getNumElements()==nCells); + + //copy the host array to the device + auto elmCenter = p_mesh->getMeshField(); + auto h_elmCenter = Kokkos::create_mirror_view(elmCenter); + for(int i=0; ip_mesh; + + //check the size + PMT_ALWAYS_ASSERT(p_mesh->getNumElements()==nCells); + + //copy the device to host + auto elmCenter = p_mesh->getMeshField(); + auto h_elmCenter = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), elmCenter); + for(int i=0; ip_mesh; + + PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); + //copy the host array to the device + auto dualArea = p_mesh->getMeshField(); + auto h_dualArea = Kokkos::create_mirror_view(dualArea); + for(int i=0; ip_mesh; + + PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); + //copy the device to host + auto dualArea = p_mesh->getMeshField(); + auto h_dualArea = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), dualArea); + for(int i=0; ip_mesh; + + //check the size + PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); + + //copy the host array to the device + auto coordsArray = p_mesh->getMeshField(); + auto h_coordsArray = Kokkos::create_mirror_view(coordsArray); + for(int i=0; ip_mesh; + + //check the size + PMT_ALWAYS_ASSERT(p_mesh->getNumVertices() == nVertices); + + //copy the device array to the host + auto coordsArray = p_mesh->getMeshField(); + auto h_coordsArray = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(),coordsArray); + for(int i=0; ip_mesh; + + //check the size + PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); + + //copy the host array to the device + auto coordsArray = p_mesh->getMeshField(); + auto h_coordsArray = Kokkos::create_mirror_view(coordsArray); + for(int i=0; ip_mesh; + + //check the size + PMT_ALWAYS_ASSERT(p_mesh->getNumVertices() == nVertices); + + //copy the device array to the host + auto coordsArray = p_mesh->getMeshField(); + auto h_coordsArray = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(),coordsArray); + for(int i=0; ip_mesh; + + //check the size + PMT_ALWAYS_ASSERT(p_mesh->getNumElements()==nCells); + + //copy the host array to the device + auto coordsArray = p_mesh->getMeshField(); + auto h_coordsArray = Kokkos::create_mirror_view(coordsArray); + kkViewHostU arrayHost(elmMass, nCells); + for(int i=0; ip_mesh; + + //check the size + PMT_ALWAYS_ASSERT(p_mesh->getNumElements() == nCells); + + //copy the device array to the host + auto coordsArray = p_mesh->getMeshField(); + auto h_coordsArray = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(),coordsArray); + for(int i=0; ip_mesh; + kkViewHostU arrayHost(array,nComps,nVertices); + Kokkos::View array_d("meshVelIncrDevice",nComps,nVertices); + Kokkos::deep_copy(array_d, arrayHost); + + auto vtxField = p_mesh->getMeshField(); + + //check the size + PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); + PMT_ALWAYS_ASSERT(static_cast(nVertices*vec2d_nEntries)==vtxField.size()); + + //copy the host array to the device + Kokkos::parallel_for("set mesh dispIncr", nVertices, KOKKOS_LAMBDA(const int iVtx){ + vtxField(iVtx,0) = array_d(0,iVtx); + vtxField(iVtx,1) = array_d(1,iVtx); + }); + pumipic::RecordTime("PolyMPO_setMeshVtxOnSurfVelIncr", timer.seconds()); +} + +void polympo_setMeshVtxOnSurfDispIncr_f(MPMesh_ptr p_mpmesh, const int nComps, const int nVertices, const double* array) { + Kokkos::Timer timer; + //check mpMesh is valid + checkMPMeshValid(p_mpmesh); + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + kkViewHostU arrayHost(array,nComps,nVertices); + Kokkos::View array_d("meshDispIncrDevice",nComps,nVertices); + Kokkos::deep_copy(array_d, arrayHost); + + auto vtxField = p_mesh->getMeshField(); + + //check the size + PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); + PMT_ALWAYS_ASSERT(static_cast(nVertices*vec2d_nEntries)==vtxField.size()); + + //copy the host array to the device + Kokkos::parallel_for("set mesh dispIncr", nVertices, KOKKOS_LAMBDA(const int iVtx){ + vtxField(iVtx,0) = array_d(0,iVtx); + vtxField(iVtx,1) = array_d(1,iVtx); + }); + pumipic::RecordTime("PolyMPO_setMeshVtxOnSurfDispIncr", timer.seconds()); +} + + +void polympo_getMeshVtxOnSurfVeloIncr_f(MPMesh_ptr p_mpmesh, const int nComps, const int nVertices, double* array) { + //check mpMesh is valid + checkMPMeshValid(p_mpmesh); + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + kkDbl2dViewHostU arrayHost(array,nComps,nVertices); + Kokkos::View array_d("meshVelIncrDevice",nComps,nVertices); + + auto vtxField = p_mesh->getMeshField(); + + //check the size + PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); + PMT_ALWAYS_ASSERT(p_mesh->getNumVertices() == nVertices); + PMT_ALWAYS_ASSERT(static_cast(nVertices*vec2d_nEntries)==vtxField.size()); + + //copy the device array to the host + Kokkos::parallel_for("get mesh dispIncr", nVertices, KOKKOS_LAMBDA(const int iVtx){ + array_d(0,iVtx) = vtxField(iVtx,0); + array_d(1,iVtx) = vtxField(iVtx,1); + }); + Kokkos::deep_copy(arrayHost, array_d); +} + +void polympo_getMeshVtxOnSurfDispIncr_f(MPMesh_ptr p_mpmesh, const int nComps, const int nVertices, double* array) { + //check mpMesh is valid + checkMPMeshValid(p_mpmesh); + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + kkDbl2dViewHostU arrayHost(array,nComps,nVertices); + Kokkos::View array_d("meshDispIncrDevice",nComps,nVertices); + + auto vtxField = p_mesh->getMeshField(); + + //check the size + PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); + PMT_ALWAYS_ASSERT(p_mesh->getNumVertices() == nVertices); + PMT_ALWAYS_ASSERT(static_cast(nVertices*vec2d_nEntries)==vtxField.size()); + + //copy the device array to the host + Kokkos::parallel_for("get mesh dispIncr", nVertices, KOKKOS_LAMBDA(const int iVtx){ + array_d(0,iVtx) = vtxField(iVtx,0); + array_d(1,iVtx) = vtxField(iVtx,1); + }); + Kokkos::deep_copy(arrayHost, array_d); +} + +bool polympo_push1P_f(MPMesh_ptr p_mpmesh){ + checkMPMeshValid(p_mpmesh); + bool is_migrating=((polyMPO::MPMesh*)p_mpmesh)->push1P(); + return is_migrating; +} + +void polympo_push_ahead_f(MPMesh_ptr p_mpmesh){ + checkMPMeshValid(p_mpmesh); + ((polyMPO::MPMesh*)p_mpmesh)->push_ahead(); +} + +void polympo_push_swap_f(MPMesh_ptr p_mpmesh){ + checkMPMeshValid(p_mpmesh); + ((polyMPO::MPMesh*)p_mpmesh)->push_swap(); +} + +void polympo_push_swap_pos_f(MPMesh_ptr p_mpmesh){ + checkMPMeshValid(p_mpmesh); + ((polyMPO::MPMesh*)p_mpmesh)->push_swap_pos(); +} + +void polympo_push_f(MPMesh_ptr p_mpmesh){ + checkMPMeshValid(p_mpmesh); + ((polyMPO::MPMesh*)p_mpmesh)->push(); +} + +//TODO skeleton of reconstruction functions +void polympo_setReconstructionOfMass_f(MPMesh_ptr p_mpmesh, const int order, const int meshEntType){ + checkMPMeshValid(p_mpmesh); + auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); + polyMPO::MeshFieldType type = static_cast(meshEntType); + if (type == polyMPO::MeshFType_VtxBased) + mpmesh->setReconstructSlice(order, type); + if (type == polyMPO::MeshFType_ElmBased) + mpmesh->setReconstructSlice(order, type); +} + +void polympo_setReconstructionOfVel_f(MPMesh_ptr p_mpmesh, const int order, const int meshEntType){ + checkMPMeshValid(p_mpmesh); + auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); + polyMPO::MeshFieldType type = static_cast(meshEntType); + if (type == polyMPO::MeshFType_VtxBased) + mpmesh->setReconstructSlice(order, type); + else { + std::cerr << "Error: This reconstruction is not supported\n"; + exit(1); + } +} + +void polympo_setReconstructionOfStrainRate_f(MPMesh_ptr p_mpmesh, const int order, const int meshEntType){ + checkMPMeshValid(p_mpmesh); + std::cerr << "Error: This routine is not implemented yet\n"; + exit(1); + (void)order; + (void)meshEntType; +} + +void polympo_setReconstructionOfStress_f(MPMesh_ptr p_mpmesh, const int order, const int meshEntType){ + checkMPMeshValid(p_mpmesh); + std::cerr << "Error: This routine is not implemented yet\n"; + exit(1); + (void)order; + (void)meshEntType; +} + +//With MPI communication done via MPAS +void polympo_vtxSubAssemblyIceArea_f(MPMesh_ptr p_mpmesh, int vtxPerElm, int nCellsPlus1, int comp, double* array){ + checkMPMeshValid(p_mpmesh); + auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + PMT_ALWAYS_ASSERT(vtxPerElm <= maxVtxsPerElm); + PMT_ALWAYS_ASSERT(nCellsPlus1 == p_mesh->getNumElements()+1); + PMT_ALWAYS_ASSERT(comp == 0 || comp== 1); //either first or second component + mpmesh->subAssemblyVtx1(vtxPerElm, nCellsPlus1, comp, array); +} + +void polympo_vtxSubAssemblyVelocity_f(MPMesh_ptr p_mpmesh, int vtxPerElm, int nCellsPlus1, int comp, double* array){ + checkMPMeshValid(p_mpmesh); + auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + PMT_ALWAYS_ASSERT(vtxPerElm <= maxVtxsPerElm); + PMT_ALWAYS_ASSERT(nCellsPlus1 == p_mesh->getNumElements()+1); + PMT_ALWAYS_ASSERT(comp == 0 || comp== 1); //either first or second component + mpmesh->subAssemblyVtx1(vtxPerElm, nCellsPlus1, comp, array); +} + +void polympo_subAssemblyCoeffs_f(MPMesh_ptr p_mpmesh, int vtxPerElm, int nCellsPlus1, double* m11, double* m12, double* m13, double* m14, + double* m22, double* m23, double* m24, + double* m33, double* m34, + double* m44){ + checkMPMeshValid(p_mpmesh); + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + PMT_ALWAYS_ASSERT(vtxPerElm <= maxVtxsPerElm); + PMT_ALWAYS_ASSERT(nCellsPlus1 == p_mesh->getNumElements()+1); + auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); + mpmesh->subAssemblyCoeffs(vtxPerElm, nCellsPlus1, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44); +} + +void polympo_regularize_and_solve_matrix_f(MPMesh_ptr p_mpmesh, int nVerticesPlus1, double* m11, double* m12, double* m13, double* m14, + double* m22, double* m23, double* m24, + double* m33, double* m34, + double* m44){ + checkMPMeshValid(p_mpmesh); + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + PMT_ALWAYS_ASSERT(nVerticesPlus1 == p_mesh->getNumVertices()+1); + auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); + mpmesh->solveMatrixAndRegularize(nVerticesPlus1, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44); +} + +void polympo_applyReconstruction_f(MPMesh_ptr p_mpmesh){ + checkMPMeshValid(p_mpmesh); + auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); + mpmesh->reconstructSlices(); +} + +void polympo_setOwningProc_f(MPMesh_ptr p_mpmesh, const int nCells, const int* array){ + checkMPMeshValid(p_mpmesh); + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + PMT_ALWAYS_ASSERT(p_mesh->meshEditable()); + kkViewHostU arrayHost(array,nCells); + + //check the size + PMT_ALWAYS_ASSERT(nCells == p_mesh->getNumElements()); + + Kokkos::View owningProc("owningProc",nCells); + Kokkos::deep_copy(owningProc, arrayHost); + p_mesh->setOwningProc(owningProc); +} + +void polympo_setElmGlobal_f(MPMesh_ptr p_mpmesh, const int nCells, const int* array){ + checkMPMeshValid(p_mpmesh); + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + Kokkos::View arrayHost("arrayHost", nCells); + for (int i = 0; i < nCells; i++) { + arrayHost(i) = array[i] - 1; // TODO right now elmID offset is set after MPs initialized + } + //check the size + PMT_ALWAYS_ASSERT(nCells == p_mesh->getNumElements()); + + Kokkos::View elmGlobal("elmGlobal",nCells); + Kokkos::deep_copy(elmGlobal, arrayHost); + p_mesh->setElmGlobal(elmGlobal); +} + +void polympo_setVtxGlobal_f(MPMesh_ptr p_mpmesh, const int nVertices, const int* array){ + checkMPMeshValid(p_mpmesh); + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + PMT_ALWAYS_ASSERT(nVertices==p_mesh->getNumVertices()); + Kokkos::View arrayHost("arrayHost", nVertices); + for (int i = 0; i < nVertices; i++) { + arrayHost(i) = array[i] - 1; // TODO right now elmID offset is set after MPs initialized + } + + Kokkos::View vtxGlobal("vtxGlobal",nVertices); + Kokkos::deep_copy(vtxGlobal, arrayHost); + p_mesh->setVtxGlobal(vtxGlobal); +} + +int polympo_getMPCount_f(MPMesh_ptr p_mpmesh) { + auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; + return p_MPs->getCount(); +} + +void polympo_enableTiming_f(){ + pumipic::EnableTiming(); +} + +void polympo_summarizeTime_f(){ + pumipic::SummarizeTime(); +} + +void polympo_setTimingVerbosity_f(int v){ + pumipic::SetTimingVerbosity(v); +} + diff --git a/src/pmpo_MPMesh.cpp b/src/pmpo_MPMesh.cpp index 73c8500..ba02a82 100644 --- a/src/pmpo_MPMesh.cpp +++ b/src/pmpo_MPMesh.cpp @@ -331,7 +331,7 @@ void MPMesh::push_ahead(){ //Interpolates latitude longitude increments and mesh velocity increments to //MP positions - sphericalInterpolation1(*this); + sphericalInterpolationDispVelIncr(*this); //Push the MPs p_MPs->updateRotLatLonAndXYZ2Tgt(p_mesh->getSphereRadius()); diff --git a/src/pmpo_MPMesh.hpp b/src/pmpo_MPMesh.hpp index 304833f..36e9056 100644 --- a/src/pmpo_MPMesh.hpp +++ b/src/pmpo_MPMesh.hpp @@ -66,16 +66,16 @@ class MPMesh{ template void assemblyVtx1(); template - void subAssemblyVtx1(int size1, int size2, int comp, double* array); + void subAssemblyVtx1(int vtxPerElm, int nCellsPlus1, int comp, double* array); - void subAssemblyCoeffs(int dim1, int dim2, double* m11, double* m12, double* m13, double* m14, - double* m22, double* m23, double* m24, - double* m33, double* m34, - double* m44); - void solveMatrixAndRegularize(int dim1, double* m11, double* m12, double* m13, double* m14, - double* m22, double* m23, double* m24, - double* m33, double* m34, - double* m44); + void subAssemblyCoeffs(int vtxPerElm, int nCellsPlus1, double* m11, double* m12, double* m13, double* m14, + double* m22, double* m23, double* m24, + double* m33, double* m34, + double* m44); + void solveMatrixAndRegularize(int nVerticesPlus1, double* m11, double* m12, double* m13, double* m14, + double* m22, double* m23, double* m24, + double* m33, double* m34, + double* m44); template void setReconstructSlice(int order, MeshFieldType type); diff --git a/src/pmpo_MPMesh_assembly.hpp b/src/pmpo_MPMesh_assembly.hpp index ae1cf70..ee26abc 100644 --- a/src/pmpo_MPMesh_assembly.hpp +++ b/src/pmpo_MPMesh_assembly.hpp @@ -210,10 +210,10 @@ void MPMesh::computeMatricesAndSolve(){ } //Method 2 for coefficients -void MPMesh::subAssemblyCoeffs(int dim1, int dim2, double* m11, double* m12, double* m13, double* m14, - double* m22, double* m23, double* m24, - double* m33, double* m34, - double* m44){ +void MPMesh::subAssemblyCoeffs(int vtxPerElm, int nCellsPlus1, double* m11, double* m12, double* m13, double* m14, + double* m22, double* m23, double* m24, + double* m33, double* m34, + double* m44){ Kokkos::Timer timer; //Material Points Information @@ -242,16 +242,16 @@ void MPMesh::subAssemblyCoeffs(int dim1, int dim2, double* m11, double* m12, dou if(p_mesh->getGeomType() == geom_spherical_surf) radius=p_mesh->getSphereRadius(); - Kokkos::View m11_d("m11", dim1, dim2); - Kokkos::View m12_d("m12", dim1, dim2); - Kokkos::View m13_d("m13", dim1, dim2); - Kokkos::View m14_d("m14", dim1, dim2); - Kokkos::View m22_d("m22", dim1, dim2); - Kokkos::View m23_d("m23", dim1, dim2); - Kokkos::View m24_d("m23", dim1, dim2); - Kokkos::View m33_d("m33", dim1, dim2); - Kokkos::View m34_d("m34", dim1, dim2); - Kokkos::View m44_d("m34", dim1, dim2); + Kokkos::View m11_d("m11", vtxPerElm, nCellsPlus1); + Kokkos::View m12_d("m12", vtxPerElm, nCellsPlus1); + Kokkos::View m13_d("m13", vtxPerElm, nCellsPlus1); + Kokkos::View m14_d("m14", vtxPerElm, nCellsPlus1); + Kokkos::View m22_d("m22", vtxPerElm, nCellsPlus1); + Kokkos::View m23_d("m23", vtxPerElm, nCellsPlus1); + Kokkos::View m24_d("m23", vtxPerElm, nCellsPlus1); + Kokkos::View m33_d("m33", vtxPerElm, nCellsPlus1); + Kokkos::View m34_d("m34", vtxPerElm, nCellsPlus1); + Kokkos::View m44_d("m34", vtxPerElm, nCellsPlus1); auto sub_assemble = PS_LAMBDA(const int& elm, const int& mp, const int& mask) { if(mask && (elm2Process(elm)==comm_rank)) { //if material point is 'active'/'enabled' @@ -279,16 +279,16 @@ void MPMesh::subAssemblyCoeffs(int dim1, int dim2, double* m11, double* m12, dou pumipic::RecordTime("VtxSubAssemblyComputeCoeff", timer.seconds()); Kokkos::Timer timer2; - kkDbl2dViewHostU m11_h(m11, dim1, dim2); - kkDbl2dViewHostU m12_h(m12, dim1, dim2); - kkDbl2dViewHostU m13_h(m13, dim1, dim2); - kkDbl2dViewHostU m14_h(m14, dim1, dim2); - kkDbl2dViewHostU m22_h(m22, dim1, dim2); - kkDbl2dViewHostU m23_h(m23, dim1, dim2); - kkDbl2dViewHostU m24_h(m24, dim1, dim2); - kkDbl2dViewHostU m33_h(m33, dim1, dim2); - kkDbl2dViewHostU m34_h(m34, dim1, dim2); - kkDbl2dViewHostU m44_h(m44, dim1, dim2); + kkDbl2dViewHostU m11_h(m11, vtxPerElm, nCellsPlus1); + kkDbl2dViewHostU m12_h(m12, vtxPerElm, nCellsPlus1); + kkDbl2dViewHostU m13_h(m13, vtxPerElm, nCellsPlus1); + kkDbl2dViewHostU m14_h(m14, vtxPerElm, nCellsPlus1); + kkDbl2dViewHostU m22_h(m22, vtxPerElm, nCellsPlus1); + kkDbl2dViewHostU m23_h(m23, vtxPerElm, nCellsPlus1); + kkDbl2dViewHostU m24_h(m24, vtxPerElm, nCellsPlus1); + kkDbl2dViewHostU m33_h(m33, vtxPerElm, nCellsPlus1); + kkDbl2dViewHostU m34_h(m34, vtxPerElm, nCellsPlus1); + kkDbl2dViewHostU m44_h(m44, vtxPerElm, nCellsPlus1); Kokkos::deep_copy(m11_h, m11_d); Kokkos::deep_copy(m12_h, m12_d); @@ -305,33 +305,33 @@ void MPMesh::subAssemblyCoeffs(int dim1, int dim2, double* m11, double* m12, dou } //Method 2 for coefficients Solve matrix -void MPMesh::solveMatrixAndRegularize(int dim1, double* m11, double* m12, double* m13, double* m14, +void MPMesh::solveMatrixAndRegularize(int nVerticesPlus1, double* m11, double* m12, double* m13, double* m14, double* m22, double* m23, double* m24, double* m33, double* m34, double* m44){ Kokkos::Timer timer; - kkViewHostU m11_h(m11, dim1); - kkViewHostU m12_h(m12, dim1); - kkViewHostU m13_h(m13, dim1); - kkViewHostU m14_h(m14, dim1); - kkViewHostU m22_h(m22, dim1); - kkViewHostU m23_h(m23, dim1); - kkViewHostU m24_h(m24, dim1); - kkViewHostU m33_h(m33, dim1); - kkViewHostU m34_h(m34, dim1); - kkViewHostU m44_h(m44, dim1); + kkViewHostU m11_h(m11, nVerticesPlus1); + kkViewHostU m12_h(m12, nVerticesPlus1); + kkViewHostU m13_h(m13, nVerticesPlus1); + kkViewHostU m14_h(m14, nVerticesPlus1); + kkViewHostU m22_h(m22, nVerticesPlus1); + kkViewHostU m23_h(m23, nVerticesPlus1); + kkViewHostU m24_h(m24, nVerticesPlus1); + kkViewHostU m33_h(m33, nVerticesPlus1); + kkViewHostU m34_h(m34, nVerticesPlus1); + kkViewHostU m44_h(m44, nVerticesPlus1); - Kokkos::View m11_d("m11", dim1); - Kokkos::View m12_d("m12", dim1); - Kokkos::View m13_d("m13", dim1); - Kokkos::View m14_d("m14", dim1); - Kokkos::View m22_d("m22", dim1); - Kokkos::View m23_d("m23", dim1); - Kokkos::View m24_d("m24", dim1); - Kokkos::View m33_d("m33", dim1); - Kokkos::View m34_d("m34", dim1); - Kokkos::View m44_d("m44", dim1); + Kokkos::View m11_d("m11", nVerticesPlus1); + Kokkos::View m12_d("m12", nVerticesPlus1); + Kokkos::View m13_d("m13", nVerticesPlus1); + Kokkos::View m14_d("m14", nVerticesPlus1); + Kokkos::View m22_d("m22", nVerticesPlus1); + Kokkos::View m23_d("m23", nVerticesPlus1); + Kokkos::View m24_d("m24", nVerticesPlus1); + Kokkos::View m33_d("m33", nVerticesPlus1); + Kokkos::View m34_d("m34", nVerticesPlus1); + Kokkos::View m44_d("m44", nVerticesPlus1); Kokkos::deep_copy(m11_d, m11_h); Kokkos::deep_copy(m12_d, m12_h); @@ -347,9 +347,9 @@ void MPMesh::solveMatrixAndRegularize(int dim1, double* m11, double* m12, double Kokkos::Timer timer2; auto dual_triangle_area=p_mesh->getMeshField(); - Kokkos::View VtxCoeffs("VtxCoeffs", dim1); + Kokkos::View VtxCoeffs("VtxCoeffs", nVerticesPlus1); double radius=p_mesh->getSphereRadius(); - Kokkos::parallel_for("fill", dim1, KOKKOS_LAMBDA(const int vtx){ + Kokkos::parallel_for("fill", nVerticesPlus1, KOKKOS_LAMBDA(const int vtx){ Vec4d v0 = {m11_d(vtx), m12_d(vtx), m13_d(vtx), m14_d(vtx)}; Vec4d v1 = {m12_d(vtx), m22_d(vtx), m23_d(vtx), m24_d(vtx)}; Vec4d v2 = {m13_d(vtx), m23_d(vtx), m33_d(vtx), m34_d(vtx)}; @@ -378,7 +378,7 @@ void MPMesh::solveMatrixAndRegularize(int dim1, double* m11, double* m12, double //Method2 template -void MPMesh::subAssemblyVtx1(int size1, int size2, int comp, double* array) { +void MPMesh::subAssemblyVtx1(int vtxPerElm, int nCellsPlus1, int comp, double* array) { Kokkos::Timer timer; auto VtxCoeffs=this->precomputedVtxCoeffs; @@ -402,7 +402,7 @@ void MPMesh::subAssemblyVtx1(int size1, int size2, int comp, double* array) { double radius=p_mesh->getSphereRadius(); - Kokkos::View array_d("reconstructedIceArea", size1, size2); + Kokkos::View array_d("reconstructedIceArea", vtxPerElm, nCellsPlus1); auto sub_assemble = PS_LAMBDA(const int& elm, const int& mp, const int& mask) { if(mask && (elm2Process(elm)==comm_rank)) { int nVtxE = elm2VtxConn(elm,0); //number of vertices bounding the element @@ -426,7 +426,7 @@ void MPMesh::subAssemblyVtx1(int size1, int size2, int comp, double* array) { pumipic::RecordTime("polyMPOsubAssemblyFieldCompute", timer.seconds()); Kokkos::Timer timer2; - kkDbl2dViewHostU arrayHost(array, size1, size2); + kkDbl2dViewHostU arrayHost(array, vtxPerElm, nCellsPlus1); Kokkos::deep_copy(arrayHost, array_d); pumipic::RecordTime("PolyMPOsubAssemblyFieldGet", timer2.seconds()); } diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index 92fa3fc..eea7dec 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -61,7 +61,7 @@ void polympo_setMPICommunicator_f(MPMesh_ptr p_mpmesh, MPI_Fint fcomm){ void polympo_createMPs_f(MPMesh_ptr p_mpmesh, const int numElms, - const int numMPs, // total nof of MPs which is >= no of active MPs + const int numMPs, // total number of MPs which is >= no of active MPs int* mpsPerElm, const int* mp2Elm, const int* isMPActive) { @@ -136,7 +136,7 @@ void polympo_createMPs_f(MPMesh_ptr p_mpmesh, } void polympo_startRebuildMPs_f(MPMesh_ptr p_mpmesh, - const int numMPs, // Total # MPs which is GREATER than or equal to number of active MPs + const int numMPs, // Total MPs which is GREATER than or equal to number of active MPs const int* allMP2Elm, const int* addedMPMask) { checkMPMeshValid(p_mpmesh); @@ -199,9 +199,9 @@ void polympo_startRebuildMPs_f(MPMesh_ptr p_mpmesh, void polympo_startRebuildMPs2_f(MPMesh_ptr p_mpmesh, const int sizeMP2elm, - const int* elem_ids, - const int nMPs_delete, - const int nMPs_add, + const int* elem_ids, + const int nMPs_delete, + const int nMPs_add, int* recvMPs_elm, int* recvMPs_ids) { @@ -237,8 +237,6 @@ void polympo_startRebuildMPs2_f(MPMesh_ptr p_mpmesh, pumipic::RecordTime("polympo_startRebuildMPs2_f", timer.seconds()); } - - void polympo_finishRebuildMPs_f(MPMesh_ptr p_mpmesh){ Kokkos::Timer timer; checkMPMeshValid(p_mpmesh); @@ -948,6 +946,7 @@ void polympo_setMeshDualTriangleArea_f(MPMesh_ptr p_mpmesh, const int nVertices, checkMPMeshValid(p_mpmesh); auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); //copy the host array to the device auto dualArea = p_mesh->getMeshField(); auto h_dualArea = Kokkos::create_mirror_view(dualArea); @@ -963,6 +962,7 @@ void polympo_getMeshDualTriangleArea_f(MPMesh_ptr p_mpmesh, const int nVertices, checkMPMeshValid(p_mpmesh); auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); //copy the device to host auto dualArea = p_mesh->getMeshField(); auto h_dualArea = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), dualArea); @@ -1238,52 +1238,50 @@ void polympo_setReconstructionOfStress_f(MPMesh_ptr p_mpmesh, const int order, c (void)meshEntType; } - //With MPI communication done via MPAS -void polympo_vtxSubAssemblyIceArea_f(MPMesh_ptr p_mpmesh, int size1, int size2, int comp, double* array){ +void polympo_vtxSubAssemblyIceArea_f(MPMesh_ptr p_mpmesh, int vtxPerElm, int nCellsPlus1, int comp, double* array){ checkMPMeshValid(p_mpmesh); auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - PMT_ALWAYS_ASSERT(size1 <= maxVtxsPerElm); - PMT_ALWAYS_ASSERT(size2 == p_mesh->getNumElements()+1); - PMT_ALWAYS_ASSERT(comp == 0 || comp== 1); - mpmesh->subAssemblyVtx1(size1, size2, comp, array); + PMT_ALWAYS_ASSERT(vtxPerElm <= maxVtxsPerElm); + PMT_ALWAYS_ASSERT(nCellsPlus1 == p_mesh->getNumElements()+1); + PMT_ALWAYS_ASSERT(comp == 0 || comp== 1); //either first or second component + mpmesh->subAssemblyVtx1(vtxPerElm, nCellsPlus1, comp, array); } -void polympo_vtxSubAssemblyVelocity_f(MPMesh_ptr p_mpmesh, int size1, int size2, int comp, double* array){ +void polympo_vtxSubAssemblyVelocity_f(MPMesh_ptr p_mpmesh, int vtxPerElm, int nCellsPlus1, int comp, double* array){ checkMPMeshValid(p_mpmesh); auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - PMT_ALWAYS_ASSERT(size1 <= maxVtxsPerElm); - PMT_ALWAYS_ASSERT(size2 == p_mesh->getNumElements()+1); - PMT_ALWAYS_ASSERT(comp == 0 || comp== 1); - mpmesh->subAssemblyVtx1(size1, size2, comp, array); + PMT_ALWAYS_ASSERT(vtxPerElm <= maxVtxsPerElm); + PMT_ALWAYS_ASSERT(nCellsPlus1 == p_mesh->getNumElements()+1); + PMT_ALWAYS_ASSERT(comp == 0 || comp== 1); //either first or second component + mpmesh->subAssemblyVtx1(vtxPerElm, nCellsPlus1, comp, array); } -void polympo_subAssemblyCoeffs_f(MPMesh_ptr p_mpmesh, int dim1, int dim2, double* m11, double* m12, double* m13, double* m14, - double* m22, double* m23, double* m24, - double* m33, double* m34, - double* m44){ +void polympo_subAssemblyCoeffs_f(MPMesh_ptr p_mpmesh, int vtxPerElm, int nCellsPlus1, double* m11, double* m12, double* m13, double* m14, + double* m22, double* m23, double* m24, + double* m33, double* m34, + double* m44){ checkMPMeshValid(p_mpmesh); auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - PMT_ALWAYS_ASSERT(dim1 <= maxVtxsPerElm); - PMT_ALWAYS_ASSERT(dim2 == p_mesh->getNumElements()+1); + PMT_ALWAYS_ASSERT(vtxPerElm <= maxVtxsPerElm); + PMT_ALWAYS_ASSERT(nCellsPlus1 == p_mesh->getNumElements()+1); auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); - mpmesh->subAssemblyCoeffs(dim1, dim2, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44); + mpmesh->subAssemblyCoeffs(vtxPerElm, nCellsPlus1, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44); } -void polympo_regularize_and_solve_matrix_f(MPMesh_ptr p_mpmesh, int dim1, double* m11, double* m12, double* m13, double* m14, +void polympo_regularize_and_solve_matrix_f(MPMesh_ptr p_mpmesh, int nVerticesPlus1, double* m11, double* m12, double* m13, double* m14, double* m22, double* m23, double* m24, double* m33, double* m34, double* m44){ checkMPMeshValid(p_mpmesh); + auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + PMT_ALWAYS_ASSERT(nVerticesPlus1 == p_mesh->getNumVertices()+1); auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); - mpmesh->solveMatrixAndRegularize(dim1, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44); - + mpmesh->solveMatrixAndRegularize(nVerticesPlus1, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44); } - - void polympo_applyReconstruction_f(MPMesh_ptr p_mpmesh){ checkMPMeshValid(p_mpmesh); auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); @@ -1322,6 +1320,7 @@ void polympo_setElmGlobal_f(MPMesh_ptr p_mpmesh, const int nCells, const int* ar void polympo_setVtxGlobal_f(MPMesh_ptr p_mpmesh, const int nVertices, const int* array){ checkMPMeshValid(p_mpmesh); auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; + PMT_ALWAYS_ASSERT(nVertices==p_mesh->getNumVertices()); Kokkos::View arrayHost("arrayHost", nVertices); for (int i = 0; i < nVertices; i++) { arrayHost(i) = array[i] - 1; // TODO right now elmID offset is set after MPs initialized diff --git a/src/pmpo_c.h b/src/pmpo_c.h index c05cbf0..1f066f5 100644 --- a/src/pmpo_c.h +++ b/src/pmpo_c.h @@ -113,13 +113,13 @@ void polympo_setReconstructionOfStress_f(MPMesh_ptr p_mpmesh, const int order, c void polympo_applyReconstruction_f(MPMesh_ptr p_mpmesh); //Reconstruction using MPAS -void polympo_vtxSubAssemblyIceArea_f(MPMesh_ptr p_mpmesh, int size1, int size2, int comp, double* array); -void polympo_vtxSubAssemblyVelocity_f(MPMesh_ptr p_mpmesh, int size1, int size2, int comp, double* array); -void polympo_subAssemblyCoeffs_f(MPMesh_ptr p_mpmesh, int dim1, int dim2, double* m11, double* m12, double* m13, double* m14, - double* m22, double* m23, double* m24, - double* m33, double* m34, - double* m44); -void polympo_regularize_and_solve_matrix_f(MPMesh_ptr p_mpmesh, int dim1, double* m11, double* m12, double* m13, double* m14, +void polympo_vtxSubAssemblyIceArea_f(MPMesh_ptr p_mpmesh, int vtxPerElm, int nCellsPlus1, int comp, double* array); +void polympo_vtxSubAssemblyVelocity_f(MPMesh_ptr p_mpmesh, int vtxPerElm, int nCellsPlus1, int comp, double* array); +void polympo_subAssemblyCoeffs_f(MPMesh_ptr p_mpmesh, int vtxPerElm, int nCellsPlus1, double* m11, double* m12, double* m13, double* m14, + double* m22, double* m23, double* m24, + double* m33, double* m34, + double* m44); +void polympo_regularize_and_solve_matrix_f(MPMesh_ptr p_mpmesh, int nVeticesPlus1, double* m11, double* m12, double* m13, double* m14, double* m22, double* m23, double* m24, double* m33, double* m34, double* m44); diff --git a/src/pmpo_defines.h b/src/pmpo_defines.h index 14435eb..4539a63 100644 --- a/src/pmpo_defines.h +++ b/src/pmpo_defines.h @@ -4,7 +4,7 @@ #define MP_ACTIVE 1 #define MP_DELETE -1 - +#define INVALID_ELM_ID -1 typedef void* MPMesh_ptr; //Function that receives void* and returns an int typedef int (*IntVoidFunc)(void*); diff --git a/src/pmpo_fortran.f90 b/src/pmpo_fortran.f90 index 22a7635..c5ef73e 100644 --- a/src/pmpo_fortran.f90 +++ b/src/pmpo_fortran.f90 @@ -910,38 +910,38 @@ subroutine polympo_setReconstructionOfStress(mpMesh, order, meshEntType) & end subroutine - subroutine polympo_vtxSubAssemblyIceArea(mpMesh, size1, size2, comp, array) & + subroutine polympo_vtxSubAssemblyIceArea(mpMesh, vtxPerElm, nCellsPlus1, comp, array) & bind(C, NAME='polympo_vtxSubAssemblyIceArea_f') use :: iso_c_binding type(c_ptr), value :: mpMesh - integer(c_int), value :: size1, size2, comp + integer(c_int), value :: vtxPerElm, nCellsPlus1, comp type(c_ptr), value :: array end subroutine - subroutine polympo_vtxSubAssemblyVelocity(mpMesh, size1, size2, comp, array) & + subroutine polympo_vtxSubAssemblyVelocity(mpMesh, vtxPerElm, nCellsPlus1, comp, array) & bind(C, NAME='polympo_vtxSubAssemblyVelocity_f') use :: iso_c_binding type(c_ptr), value :: mpMesh - integer(c_int), value :: size1, size2, comp + integer(c_int), value :: vtxPerElm, nCellsPlus1, comp type(c_ptr), value :: array end subroutine - subroutine polympo_subAssemblyCoeffs(mpMesh, dim1, dim2, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44) & + subroutine polympo_subAssemblyCoeffs(mpMesh, vtxPerElm, nCellsPlus1, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44) & bind(C, NAME='polympo_subAssemblyCoeffs_f') use :: iso_c_binding type(c_ptr), value :: mpMesh - integer(c_int), value :: dim1, dim2 + integer(c_int), value :: vtxPerElm, nCellsPlus1 type(c_ptr), value :: m11, m12, m13, m14, m22, m23, m24, m33, m34, m44 end subroutine - subroutine polympo_regularize_and_solve_matrix(mpMesh, dim1, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44) & + + subroutine polympo_regularize_and_solve_matrix(mpMesh, nVerticesPlus1, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44) & bind(C, NAME='polympo_regularize_and_solve_matrix_f') use :: iso_c_binding type(c_ptr), value :: mpMesh - integer(c_int), value :: dim1 + integer(c_int), value :: nVerticesPlus1 type(c_ptr), value :: m11, m12, m13, m14, m22, m23, m24, m33, m34, m44 end subroutine - !--------------------------------------------------------------------------- !> @brief directly call the reconstruct of the MP fields to mesh fields !> @param mpmesh(in/out) MPMesh object diff --git a/src/pmpo_materialPoints.cpp b/src/pmpo_materialPoints.cpp index 4037f99..0f6b6c1 100644 --- a/src/pmpo_materialPoints.cpp +++ b/src/pmpo_materialPoints.cpp @@ -14,7 +14,7 @@ pumipic::MemberTypeViews createInternalMemberViews(int numMPs, View mp2elm, View auto policy = Kokkos::RangePolicy(typename MemSpace::execution_space(), 0, numMPs); Kokkos::parallel_for("setMPinfo", policy, KOKKOS_LAMBDA(int i) { mpCurElmPos_m(i) = mp2elm(i); - mpTgtElmPos_m(i) = -1; + mpTgtElmPos_m(i) = INVALID_ELM_ID; mpStatus_m(i) = MP_ACTIVE; mpAppID_m(i) = mpAppID(i); }); diff --git a/src/pmpo_utils.hpp b/src/pmpo_utils.hpp index b9221b9..76c2a7d 100644 --- a/src/pmpo_utils.hpp +++ b/src/pmpo_utils.hpp @@ -51,36 +51,36 @@ class Vec2d { //constructors KOKKOS_INLINE_FUNCTION Vec2d():coords_{0.0, 0.0}{} - + ~Vec2d() = default; - + KOKKOS_INLINE_FUNCTION Vec2d(double x1, double x2):coords_{x1,x2}{} - + KOKKOS_INLINE_FUNCTION Vec2d(double x[2]):coords_{x[0], x[1]}{} //operators KOKKOS_INLINE_FUNCTION double operator[](int i) const { return coords_[i]; } - + KOKKOS_INLINE_FUNCTION double &operator[](int i) { return coords_[i]; } KOKKOS_INLINE_FUNCTION Vec2d operator-() { return Vec2d(-coords_[0], -coords_[1]); } - + KOKKOS_INLINE_FUNCTION Vec2d operator+(Vec2d v) { return Vec2d(coords_[0] + v[0], - coords_[1] + v[1]); } + coords_[1] + v[1]); } KOKKOS_INLINE_FUNCTION Vec2d operator-(Vec2d v) { return Vec2d(coords_[0] - v[0], - coords_[1] - v[1]); } + coords_[1] - v[1]); } KOKKOS_INLINE_FUNCTION Vec2d operator*(double scalar) const { return Vec2d(coords_[0]*scalar, - coords_[1]*scalar); } + coords_[1]*scalar); } KOKKOS_INLINE_FUNCTION double dot(Vec2d v) { return coords_[0]*v[0]+coords_[1]*v[1]; } @@ -119,12 +119,12 @@ class Vec3d{ KOKKOS_INLINE_FUNCTION Vec3d operator+(const Vec3d& v) const { - return Vec3d(coords_[0] + v.coords_[0], coords_[1] + v.coords_[1], coords_[2] + v.coords_[2]); + return Vec3d(coords_[0] + v.coords_[0], coords_[1] + v.coords_[1], coords_[2] + v.coords_[2]); } KOKKOS_INLINE_FUNCTION Vec3d operator-(const Vec3d& v) const { - return Vec3d(coords_[0] - v.coords_[0], coords_[1] - v.coords_[1], coords_[2] - v.coords_[2]); + return Vec3d(coords_[0] - v.coords_[0], coords_[1] - v.coords_[1], coords_[2] - v.coords_[2]); } KOKKOS_INLINE_FUNCTION @@ -132,24 +132,24 @@ class Vec3d{ KOKKOS_INLINE_FUNCTION Vec3d operator*(double scalar) const { - return Vec3d(coords_[0] * scalar, coords_[1] * scalar, coords_[2] * scalar); + return Vec3d(coords_[0] * scalar, coords_[1] * scalar, coords_[2] * scalar); } KOKKOS_INLINE_FUNCTION double dot(const Vec3d& v) const { - return coords_[0] * v.coords_[0] + coords_[1] * v.coords_[1] + coords_[2] * v.coords_[2]; + return coords_[0] * v.coords_[0] + coords_[1] * v.coords_[1] + coords_[2] * v.coords_[2]; } KOKKOS_INLINE_FUNCTION Vec3d cross(const Vec3d& v) const { - return Vec3d(coords_[1] * v.coords_[2] - coords_[2] * v.coords_[1], - coords_[2] * v.coords_[0] - coords_[0] * v.coords_[2], - coords_[0] * v.coords_[1] - coords_[1] * v.coords_[0]); + return Vec3d(coords_[1] * v.coords_[2] - coords_[2] * v.coords_[1], + coords_[2] * v.coords_[0] - coords_[0] * v.coords_[2], + coords_[0] * v.coords_[1] - coords_[1] * v.coords_[0]); } KOKKOS_INLINE_FUNCTION double magnitude() const { - return std::sqrt(coords_[0] * coords_[0] + coords_[1] * coords_[1] + coords_[2] * coords_[2]); + return std::sqrt(coords_[0] * coords_[0] + coords_[1] * coords_[1] + coords_[2] * coords_[2]); } }; @@ -183,7 +183,7 @@ class Vec4d{ Vec4d operator+(const Vec4d& v) const { return Vec4d(coords_[0] + v.coords_[0], coords_[1] + v.coords_[1], coords_[2] + v.coords_[2], coords_[3] + v.coords_[3]); - } + } KOKKOS_INLINE_FUNCTION Vec4d operator-(const Vec4d& v) const { @@ -301,145 +301,144 @@ void initArray(Vec2d* arr, int n, Vec2d fill){ KOKKOS_INLINE_FUNCTION void CholeskySolve4d_UnitRHS(Matrix4d& A, double* x){ - double a_00=A(0,0); - if (A(0,0)==0){ - x[0]=0; - x[1]=0; - x[2]=0; - x[3]=0; - return; - } + double a_00=A(0,0); + if (A(0,0)==0){ + x[0]=0; + x[1]=0; + x[2]=0; + x[3]=0; + return; + } - A(0,0) = std::sqrt(A(0,0)); - A(0,1) /= A(0,0); - A(0,2) /= A(0,0); - A(0,3) /= A(0,0); + A(0,0) = std::sqrt(A(0,0)); + A(0,1) /= A(0,0); + A(0,2) /= A(0,0); + A(0,3) /= A(0,0); - double diag = A(1,1) - A(0,1)*A(0,1); - if(diag>EPSILON){ - A(1,1)=std::sqrt(diag); - A(1,2)=(A(1,2)-A(0,1)*A(0,2))/A(1,1); - A(1,3)=(A(1,3)-A(0,1)*A(0,3))/A(1,1); - } - else{ - A(1,1)=0.0; - A(1,2)=0.0; - A(1,3)=0.0; - } + double diag = A(1,1) - A(0,1)*A(0,1); + if(diag>EPSILON){ + A(1,1)=std::sqrt(diag); + A(1,2)=(A(1,2)-A(0,1)*A(0,2))/A(1,1); + A(1,3)=(A(1,3)-A(0,1)*A(0,3))/A(1,1); + } + else{ + A(1,1)=0.0; + A(1,2)=0.0; + A(1,3)=0.0; + } - diag = A(2,2) - A(0,2)*A(0,2)-A(1,2)*A(1,2); - if(diag>EPSILON){ - A(2,2)=std::sqrt(diag); - A(2,3)=(A(2,3)-A(0,2)*A(0,3)-A(1,2)*A(1,3))/A(2,2); - } - else{ - A(2,2)=0.0; - A(2,3)=0.0; - } + diag = A(2,2) - A(0,2)*A(0,2)-A(1,2)*A(1,2); + if(diag>EPSILON){ + A(2,2)=std::sqrt(diag); + A(2,3)=(A(2,3)-A(0,2)*A(0,3)-A(1,2)*A(1,3))/A(2,2); + } + else{ + A(2,2)=0.0; + A(2,3)=0.0; + } - diag=A(3,3)-A(0,3)*A(0,3)-A(1,3)*A(1,3)-A(2,3)*A(2,3); - if(diag>EPSILON) - A(3,3)=std::sqrt(diag); - else - A(3,3)=0.0; - - if(abs(A(1,1))EPSILON) + A(3,3)=std::sqrt(diag); + else + A(3,3)=0.0; + + if(abs(A(1,1))getMeshField(); @@ -423,7 +423,7 @@ inline void sphericalInterpolation1(MPMesh& mpMesh){ } }; p_MPs->parallel_for(interpolation, "sphericalInterpolationMultiField"); - pumipic::RecordTime("PolyMPO_sphericalInterpolation1", timer.seconds()); + pumipic::RecordTime("PolyMPO_sphericalInterpolationDispVelIncr", timer.seconds()); } From 4839c60fb7060b58fcbff3231025ded38975de4d Mon Sep 17 00:00:00 2001 From: Nath Date: Wed, 13 Aug 2025 18:53:19 -0400 Subject: [PATCH 40/42] PR changes2_v1 --- "\\" | 1350 ---------------------------------------------------------- 1 file changed, 1350 deletions(-) delete mode 100644 "\\" diff --git "a/\\" "b/\\" deleted file mode 100644 index f315b4a..0000000 --- "a/\\" +++ /dev/null @@ -1,1350 +0,0 @@ -#include "pmpo_createTestMPMesh.hpp" -#include "pmpo_defines.h" -#include "pmpo_c.h" -#include "pmpo_MPMesh_assembly.hpp" -#include - -namespace{ - std::vector p_mpmeshes;////store the p_mpmeshes that is legal - - void checkMPMeshValid(MPMesh_ptr p_mpmesh){ - auto p_mpmeshIter = std::find(p_mpmeshes.begin(),p_mpmeshes.end(),p_mpmesh); - PMT_ALWAYS_ASSERT(p_mpmeshIter != p_mpmeshes.end()); - } -} - -void polympo_initialize_f() { - int isMPIInit; - MPI_Initialized(&isMPIInit); - PMT_ALWAYS_ASSERT(isMPIInit); - Kokkos::initialize(); -} - -void polympo_finalize_f() { - Kokkos::finalize(); -} - -MPMesh_ptr polympo_createMPMesh_f(const int testMeshOption, const int testMPOption) { - polyMPO::Mesh* p_mesh; - if(testMeshOption){ - int replicateFactor = 1; - p_mesh = polyMPO::initTestMesh(testMeshOption, replicateFactor); - }else{ - p_mesh = new polyMPO::Mesh(); - } - polyMPO::MaterialPoints* p_mps; - if(testMPOption){ - PMT_ALWAYS_ASSERT(testMeshOption >= 1); - p_mps = polyMPO::initTestMPs(p_mesh, testMPOption); - }else{ - p_mps = new polyMPO::MaterialPoints(); - } - MPMesh_ptr p_mpMeshReturn = (MPMesh_ptr) new polyMPO::MPMesh(p_mesh, p_mps); - p_mpmeshes.push_back(p_mpMeshReturn); - return p_mpMeshReturn; -} - -void polympo_deleteMPMesh_f(MPMesh_ptr p_mpmesh) { - //check mpMesh is valid - auto p_mpmeshIter = std::find(p_mpmeshes.begin(),p_mpmeshes.end(),p_mpmesh); - PMT_ALWAYS_ASSERT(p_mpmeshIter != p_mpmeshes.end()); - p_mpmeshes.erase(p_mpmeshIter); - delete (polyMPO::MPMesh*)p_mpmesh; -} - -void polympo_setMPICommunicator_f(MPMesh_ptr p_mpmesh, MPI_Fint fcomm){ - checkMPMeshValid(p_mpmesh); - MPI_Comm comm = MPI_Comm_f2c(fcomm); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - p_MPs->setMPIComm(comm); -} - -void polympo_createMPs_f(MPMesh_ptr p_mpmesh, - const int numElms, - const int numMPs, // total number of MPs which is >= no of active MPs - int* mpsPerElm, - const int* mp2Elm, - const int* isMPActive) { - checkMPMeshValid(p_mpmesh); - //the mesh must be fixed/set before adding MPs - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - PMT_ALWAYS_ASSERT(!p_mesh->meshEditable()); - PMT_ALWAYS_ASSERT(p_mesh->getNumElements() == numElms); - - //Find the total no of MPs across all ranks - //And loop over all MPs and find the smallest element id associated across a MP - int numActiveMPs = 0; - int minElmID = INT_MAX; - for(int i = 0; i < numMPs; i++) { - if(isMPActive[i] == MP_ACTIVE) { - numActiveMPs++; - if(mp2Elm[i] < minElmID) - minElmID = mp2Elm[i]; - } - } - int globalNumActiveMPs = 0; - int globalMinElmID; - MPI_Allreduce(&numActiveMPs, &globalNumActiveMPs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - MPI_Allreduce(&minElmID, &globalMinElmID, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD); - PMT_ALWAYS_ASSERT(globalNumActiveMPs>0); - - //Loop over all mesh elements 0,1,... and find the first element that has an associated MP - int firstElmWithMPs=INT_MAX; - for (int i=0; i active_mpIDs(numMPs); - std::vector active_mp2Elm(numMPs); - numActiveMPs = 0; - for(int i=0; igetElmGlobal(); - auto mpsPerElm_d = create_mirror_view_and_copy(mpsPerElm, numElms); - auto active_mp2Elm_d = create_mirror_view_and_copy(active_mp2Elm.data(), numActiveMPs); - auto active_mpIDs_d = create_mirror_view_and_copy(active_mpIDs.data(), numActiveMPs); - - delete ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - ((polyMPO::MPMesh*)p_mpmesh)->p_MPs = - new polyMPO::MaterialPoints(numElms, numActiveMPs, mpsPerElm_d, active_mp2Elm_d, active_mpIDs_d, elm2global); - - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - p_MPs->setElmIDoffset(offset); - -} - -void polympo_startRebuildMPs_f(MPMesh_ptr p_mpmesh, - const int numMPs, // Total number of MPs which is GREATER than or equal to number of active MPs - const int* allMP2Elm, - const int* addedMPMask) { - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); - - int offset = p_MPs->getElmIDoffset(); - std::vector added_mpIDs(numMPs); - std::vector added_mp2Elm(numMPs); - int numAddedMPs = 0; - for(int i=0; igetCapacity(); // pumipic expects full capacity to rebuild - Kokkos::View mp2Elm("mp2Elm", internalMPCapacity); - auto mpAppID = p_MPs->getData(); - - auto added_mp2Elm_d = create_mirror_view_and_copy(added_mp2Elm.data(), numAddedMPs); - auto added_mpIDs_d = create_mirror_view_and_copy(added_mpIDs.data(), numAddedMPs); - auto addedMPMask_d = create_mirror_view_and_copy(addedMPMask, numMPs); - auto mpMP2ElmIn_d = create_mirror_view_and_copy(allMP2Elm, numMPs); - - Kokkos::View numDeletedMPs_d("numDeletedMPs", 1); - auto setMP2Elm = PS_LAMBDA(const int&, const int& mp, const int& mask) { - if(mask) { - if (addedMPMask_d[mpAppID(mp)] == MP_ACTIVE) //two MPs can not occupy the same slot - mp2Elm(mp) = MP_DELETE; - else - mp2Elm(mp) = mpMP2ElmIn_d(mpAppID(mp)); - if (mp2Elm(mp) == MP_DELETE) - Kokkos::atomic_increment(&numDeletedMPs_d(0)); - } - }; - p_MPs->parallel_for(setMP2Elm, "setMP2Elm"); - - int numDeletedMPs = pumipic::getLastValue(numDeletedMPs_d); - PMT_ALWAYS_ASSERT(numAddedMPs > 0 || numDeletedMPs > 0); - - p_MPs->startRebuild(mp2Elm, numAddedMPs, added_mp2Elm_d, added_mpIDs_d, addedMPMask_d); - - // check mpAppID is unique (on GPUs) - if (p_MPs->getOpMode() == polyMPO::MP_DEBUG){ - mpAppID = p_MPs->getData(); - Kokkos::View mpAppIDCount("mpAppIDCount", p_MPs->getCount()); - auto checkAppIDs = PS_LAMBDA(const int&, const int& mp, const int& mask){ - if(mask) { - int prev = Kokkos::atomic_fetch_add(&mpAppIDCount(mpAppID(mp)), 1); - assert(prev == 0); - } - }; - p_MPs->parallel_for(checkAppIDs, "checkAppIDs"); - } -} - -void polympo_startRebuildMPs2_f(MPMesh_ptr p_mpmesh, - const int sizeMP2elm, - const int* elem_ids, - const int nMPs_delete, - const int nMPs_add, - int* recvMPs_elm, - int* recvMPs_ids) { - - Kokkos::Timer timer; - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - int offset = p_MPs->getElmIDoffset(); - - for (int k=0; k mp2Elm("mp2Elm", p_MPs->getCapacity()); - Kokkos::View numDeletedMPs_d("numDeletedMPs", 1); - auto mpAppID = p_MPs->getData(); - auto setMP2Elm = PS_LAMBDA(const int& e, const int& mp, const int& mask) { - if(mask) { - mp2Elm(mp) = elem_ids_d(mpAppID(mp))-offset; - if (mp2Elm(mp) == MP_DELETE){ - Kokkos::atomic_increment(&numDeletedMPs_d(0)); - } - } - }; - p_MPs->parallel_for(setMP2Elm, "setMP2Elm"); - int numDeletedMPs = pumipic::getLastValue(numDeletedMPs_d); - assert(nMPs_delete==numDeletedMPs); - p_MPs->startRebuild(mp2Elm, nMPs_add, recvMPs_elm_d, recvMPs_ids_d); - pumipic::RecordTime("polympo_startRebuildMPs2_f", timer.seconds()); -} - -void polympo_finishRebuildMPs_f(MPMesh_ptr p_mpmesh){ - Kokkos::Timer timer; - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - p_MPs->finishRebuild(); - pumipic::RecordTime("polympo_finishRebuildMPs_f", timer.seconds()); -} - -void polympo_setAppIDFunc_f(MPMesh_ptr p_mpmesh, IntVoidFunc getNext, void* appIDs) { - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - polyMPO::IntFunc getNextAppID = [getNext, appIDs]() { return getNext(appIDs); }; - p_MPs->setAppIDFunc(getNextAppID); -} - -void polympo_getMPTgtElmID_f(MPMesh_ptr p_mpmesh, - const int numMPs, - int* elmIDs){ - Kokkos::Timer timer; - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); - auto mpTgtElmID = p_MPs->getData(); - auto mpAppID = p_MPs->getData(); - auto elmIDoffset = p_MPs->getElmIDoffset(); - - kkIntViewHostU arrayHost(elmIDs,numMPs); - polyMPO::IntView mpTgtElmIDCopy("mpTgtElmIDNewValue",numMPs); - - auto setTgtElmId = PS_LAMBDA(const int&, const int& mp, const int& mask){ - if(mask){ - mpTgtElmIDCopy(mpAppID(mp)) = mpTgtElmID(mp)+elmIDoffset; - } - }; - p_MPs->parallel_for(setTgtElmId, "set mpTgtElmID"); - Kokkos::deep_copy( arrayHost, mpTgtElmIDCopy); - pumipic::RecordTime("PolyMPO_getMPTgtElmID", timer.seconds()); -} - -void polympo_getMPCurElmID_f(MPMesh_ptr p_mpmesh, - const int numMPs, - int* elmIDs){ - - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); - auto mpCurElmID = p_MPs->getData(); - auto mpAppID = p_MPs->getData(); - auto elmIDoffset = p_MPs->getElmIDoffset(); - - kkIntViewHostU arrayHost(elmIDs,numMPs); - polyMPO::IntView mpCurElmIDCopy("mpCurElmIDNewValue",numMPs); - - auto getElmId = PS_LAMBDA(const int&, const int& mp, const int& mask){ - if(mask){ - mpCurElmIDCopy(mpAppID(mp)) = mpCurElmID(mp)+elmIDoffset; - } - }; - p_MPs->parallel_for(getElmId, "get mpCurElmID"); - Kokkos::deep_copy( arrayHost, mpCurElmIDCopy); - -} - -void polympo_setMPLatLonRotatedFlag_f(MPMesh_ptr p_mpmesh, const int isRotateFlag){ - //chech validity - checkMPMeshValid(p_mpmesh); - ((polyMPO::MPMesh*)p_mpmesh)->p_MPs->setRotatedFlag(isRotateFlag>0); - -} - -void polympo_setMPPositions_f(MPMesh_ptr p_mpmesh, - const int nComps, - const int numMPs, - const double* mpPositionsIn){ - Kokkos::Timer timer; - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - PMT_ALWAYS_ASSERT(nComps == vec3d_nEntries); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); - kkViewHostU mpPositionsIn_h(mpPositionsIn,nComps,numMPs); - - if (p_MPs->rebuildOngoing()) { - p_MPs->setRebuildMPSlice(mpPositionsIn_h); - return; - } - - auto mpPositions = p_MPs->getData(); - auto mpAppID = p_MPs->getData(); - - Kokkos::View mpPositionsIn_d("mpPositionsDevice",vec3d_nEntries,numMPs); - Kokkos::deep_copy(mpPositionsIn_d, mpPositionsIn_h); - auto setPos = PS_LAMBDA(const int&, const int& mp, const int& mask){ - if(mask){ - mpPositions(mp,0) = mpPositionsIn_d(0, mpAppID(mp)); - mpPositions(mp,1) = mpPositionsIn_d(1, mpAppID(mp)); - mpPositions(mp,2) = mpPositionsIn_d(2, mpAppID(mp)); - } - }; - p_MPs->parallel_for(setPos, "setMPPositions"); - pumipic::RecordTime("PolyMPO_setMPPositions", timer.seconds()); -} - -void polympo_getMPPositions_f(MPMesh_ptr p_mpmesh, - const int nComps, - const int numMPs, - double* mpPositionsHost){ - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - PMT_ALWAYS_ASSERT(nComps == vec3d_nEntries); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); - - auto mpPositions = p_MPs->getData(); - auto mpAppID = p_MPs->getData(); - Kokkos::View mpPositionsCopy("mpPositionsCopy",vec3d_nEntries,numMPs); - auto getPos = PS_LAMBDA(const int&, const int& mp, const int& mask){ - if(mask){ - mpPositionsCopy(0,mpAppID(mp)) = mpPositions(mp,0); - mpPositionsCopy(1,mpAppID(mp)) = mpPositions(mp,1); - mpPositionsCopy(2,mpAppID(mp)) = mpPositions(mp,2); - } - }; - p_MPs->parallel_for(getPos, "getMPPositions"); - kkDbl2dViewHostU arrayHost(mpPositionsHost,nComps,numMPs); - Kokkos::deep_copy(arrayHost, mpPositionsCopy); -} - - -void polympo_setMPTgtPositions_f(MPMesh_ptr p_mpmesh, - const int nComps, - const int numMPs, - const double* mpPositionsIn){ - Kokkos::Timer timer; - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - PMT_ALWAYS_ASSERT(nComps == vec3d_nEntries); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); - kkViewHostU mpPositionsIn_h(mpPositionsIn,nComps,numMPs); - - if (p_MPs->rebuildOngoing()) { - p_MPs->setRebuildMPSlice(mpPositionsIn_h); - return; - } - - auto mpPositions = p_MPs->getData(); - auto mpAppID = p_MPs->getData(); - Kokkos::View mpPositionsIn_d("mpPositionsDevice",vec3d_nEntries,numMPs); - Kokkos::deep_copy(mpPositionsIn_d, mpPositionsIn_h); - auto setPos = PS_LAMBDA(const int&, const int& mp, const int& mask){ - if(mask){ - mpPositions(mp,0) = mpPositionsIn_d(0, mpAppID(mp)); - mpPositions(mp,1) = mpPositionsIn_d(1, mpAppID(mp)); - mpPositions(mp,2) = mpPositionsIn_d(2, mpAppID(mp)); - } - }; - p_MPs->parallel_for(setPos, "setMPPositions"); - pumipic::RecordTime("PolyMPO_setMPTgtPositions", timer.seconds()); -} - -void polympo_getMPTgtPositions_f(MPMesh_ptr p_mpmesh, - const int nComps, - const int numMPs, - double* mpPositionsHost){ - Kokkos::Timer timer; - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - PMT_ALWAYS_ASSERT(nComps == vec3d_nEntries); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); - - auto mpPositions = p_MPs->getData(); - auto mpAppID = p_MPs->getData(); - Kokkos::View mpPositionsCopy("mpPositionsCopy",vec3d_nEntries,numMPs); - auto getPos = PS_LAMBDA(const int&, const int& mp, const int& mask){ - if(mask){ - mpPositionsCopy(0,mpAppID(mp)) = mpPositions(mp,0); - mpPositionsCopy(1,mpAppID(mp)) = mpPositions(mp,1); - mpPositionsCopy(2,mpAppID(mp)) = mpPositions(mp,2); - } - }; - p_MPs->parallel_for(getPos, "getMPPositions"); - kkDbl2dViewHostU arrayHost(mpPositionsHost,nComps,numMPs); - Kokkos::deep_copy(arrayHost, mpPositionsCopy); - pumipic::RecordTime("PolyMPO_getMPTgtPositions", timer.seconds()); -} - - -void polympo_setMPRotLatLon_f(MPMesh_ptr p_mpmesh, - const int nComps, - const int numMPs, - const double* mpRotLatLonIn){ - Kokkos::Timer timer; - static int callCount = 0; - PMT_ALWAYS_ASSERT(callCount == 0); - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); - - auto mpRotLatLon = p_MPs->getData(); - auto mpAppID = p_MPs->getData(); - kkViewHostU mpRotLatLonIn_h(mpRotLatLonIn,nComps,numMPs); - Kokkos::View mpRotLatLonIn_d("mpRotLatLonDevice",vec2d_nEntries,numMPs); - Kokkos::deep_copy(mpRotLatLonIn_d, mpRotLatLonIn_h); - auto setPos = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ - if(mask){ - mpRotLatLon(mp,0) = mpRotLatLonIn_d(0, mpAppID(mp)); - mpRotLatLon(mp,1) = mpRotLatLonIn_d(1, mpAppID(mp)); - } - }; - p_MPs->parallel_for(setPos, "setMPRotLatLon"); - callCount++; - pumipic::RecordTime("PolyMPO_setMPRotLatLon", timer.seconds()); -} - -void polympo_getMPRotLatLon_f(MPMesh_ptr p_mpmesh, - const int nComps, - const int numMPs, - double* mpRotLatLonHost){ - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); - - auto mpRotLatLon = p_MPs->getData(); - auto mpAppID = p_MPs->getData(); - Kokkos::View mpRotLatLonCopy("mpRotLatLonCopy",vec2d_nEntries,numMPs); - auto getPos = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ - if(mask){ - mpRotLatLonCopy(0,mpAppID(mp)) = mpRotLatLon(mp,0); - mpRotLatLonCopy(1,mpAppID(mp)) = mpRotLatLon(mp,1); - } - }; - p_MPs->parallel_for(getPos, "getMPRotLatLon"); - kkDbl2dViewHostU arrayHost(mpRotLatLonHost,nComps,numMPs); - Kokkos::deep_copy(arrayHost, mpRotLatLonCopy); -} - - -void polympo_setMPTgtRotLatLon_f(MPMesh_ptr p_mpmesh, - const int nComps, - const int numMPs, - const double* mpRotLatLonIn){ - Kokkos::Timer timer; - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); - - auto mpRotLatLon = p_MPs->getData(); - auto mpAppID = p_MPs->getData(); - kkViewHostU mpRotLatLonIn_h(mpRotLatLonIn,nComps,numMPs); - Kokkos::View mpRotLatLonIn_d("mpRotLatLonDevice",vec2d_nEntries,numMPs); - Kokkos::deep_copy(mpRotLatLonIn_d, mpRotLatLonIn_h); - auto setPos = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ - if(mask){ - mpRotLatLon(mp,0) = mpRotLatLonIn_d(0, mpAppID(mp)); - mpRotLatLon(mp,1) = mpRotLatLonIn_d(1, mpAppID(mp)); - } - }; - p_MPs->parallel_for(setPos, "setMPTgtRotLatLon"); - pumipic::RecordTime("PolyMPO_setMPTgtRotLatLon", timer.seconds()); -} - -void polympo_getMPTgtRotLatLon_f(MPMesh_ptr p_mpmesh, - const int nComps, - const int numMPs, - double* mpRotLatLonHost){ - Kokkos::Timer timer; - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); - - auto mpRotLatLon = p_MPs->getData(); - auto mpAppID = p_MPs->getData(); - Kokkos::View mpRotLatLonCopy("mpRotLatLonCopy",vec2d_nEntries,numMPs); - auto getPos = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ - if(mask){ - mpRotLatLonCopy(0,mpAppID(mp)) = mpRotLatLon(mp,0); - mpRotLatLonCopy(1,mpAppID(mp)) = mpRotLatLon(mp,1); - } - }; - p_MPs->parallel_for(getPos, "getMPRotLatLon"); - kkDbl2dViewHostU arrayHost(mpRotLatLonHost,nComps,numMPs); - Kokkos::deep_copy(arrayHost, mpRotLatLonCopy); - pumipic::RecordTime("PolyMPO_getMPTgtRotLatLon", timer.seconds()); -} - - -void polympo_setMPMass_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpMassIn) { - Kokkos::Timer timer; - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - PMT_ALWAYS_ASSERT(nComps == 1); //TODO mp_sclr_t - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); - - auto mpMass = p_MPs->getData(); - auto mpAppID = p_MPs->getData(); - kkViewHostU mpMassIn_h(mpMassIn,nComps,numMPs); - Kokkos::View mpMassIn_d("mpMassDevice",nComps,numMPs); - Kokkos::deep_copy(mpMassIn_d, mpMassIn_h); - auto setMPMass = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ - if(mask){ - mpMass(mp,0) = mpMassIn_d(0, mpAppID(mp)); - } - }; - p_MPs->parallel_for(setMPMass, "setMPMass"); - pumipic::RecordTime("PolyMPO_setMPMass", timer.seconds()); -} - -void polympo_getMPMass_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, double* mpMassHost) { - Kokkos::Timer timer; - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - PMT_ALWAYS_ASSERT(nComps == 1); //TODO mp_sclr_t - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); - - auto mpMass = p_MPs->getData(); - auto mpAppID = p_MPs->getData(); - Kokkos::View mpMassCopy("mpMassCopy",nComps,numMPs); - auto getMPMass = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ - if(mask){ - mpMassCopy(0,mpAppID(mp)) = mpMass(mp,0); - } - }; - p_MPs->parallel_for(getMPMass, "getMPMass"); - kkDbl2dViewHostU arrayHost(mpMassHost,nComps,numMPs); - Kokkos::deep_copy(arrayHost, mpMassCopy); - pumipic::RecordTime("PolyMPO_getMPMass", timer.seconds()); -} - -void polympo_setMPVel_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpVelIn) { - Kokkos::Timer timer; - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); - - auto mpVel = p_MPs->getData(); - auto mpAppID = p_MPs->getData(); - kkViewHostU mpVelIn_h(mpVelIn,nComps,numMPs); - Kokkos::View mpVelIn_d("mpVelDevice",vec2d_nEntries,numMPs); - Kokkos::deep_copy(mpVelIn_d, mpVelIn_h); - auto setMPVel = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ - if(mask){ - mpVel(mp,0) = mpVelIn_d(0, mpAppID(mp)); - mpVel(mp,1) = mpVelIn_d(1, mpAppID(mp)); - } - }; - p_MPs->parallel_for(setMPVel, "setMPVel"); - pumipic::RecordTime("PolyMPO_setMPVel", timer.seconds()); -} - -void polympo_getMPVel_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, double* mpVelHost) { - Kokkos::Timer timer; - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - //PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); - - auto mpVel = p_MPs->getData(); - auto mpAppID = p_MPs->getData(); - Kokkos::View mpVelCopy("mpVelCopy",vec2d_nEntries,numMPs); - auto getMPVel = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ - if(mask){ - mpVelCopy(0,mpAppID(mp)) = mpVel(mp,0); - mpVelCopy(1,mpAppID(mp)) = mpVel(mp,1); - } - }; - p_MPs->parallel_for(getMPVel, "getMPVel"); - kkDbl2dViewHostU arrayHost(mpVelHost,nComps,numMPs); - Kokkos::deep_copy(arrayHost, mpVelCopy); - pumipic::RecordTime("PolyMPO_getMPVel", timer.seconds()); -} - -//TODO: implement these -void polympo_setMPStrainRate_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpStrainRateIn){ - checkMPMeshValid(p_mpmesh); - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - PMT_ALWAYS_ASSERT(nComps == 6); //TODO: mp_sym_mat3d_t - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getCount()); - PMT_ALWAYS_ASSERT(numMPs >= p_MPs->getMaxAppID()); - - auto mpStrainRate = p_MPs->getData(); - auto mpAppID = p_MPs->getData(); - kkViewHostU mpStrainRateIn_h(mpStrainRateIn,nComps,numMPs); - Kokkos::View mpStrainRateIn_d("mpStrainRateDevice",nComps,numMPs); - Kokkos::deep_copy(mpStrainRateIn_d, mpStrainRateIn_h); - auto setMPStrainRate = PS_LAMBDA(const int& elm, const int& mp, const int& mask){ - if(mask){ - mpStrainRate(mp,0) = mpStrainRateIn_d(0, mpAppID(mp)); - mpStrainRate(mp,1) = mpStrainRateIn_d(1, mpAppID(mp)); - mpStrainRate(mp,2) = mpStrainRateIn_d(2, mpAppID(mp)); - mpStrainRate(mp,3) = mpStrainRateIn_d(3, mpAppID(mp)); - mpStrainRate(mp,4) = mpStrainRateIn_d(4, mpAppID(mp)); - mpStrainRate(mp,5) = mpStrainRateIn_d(5, mpAppID(mp)); - } - }; - p_MPs->parallel_for(setMPStrainRate, "setMPStrainRate"); -} -void polympo_getMPStrainRate_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, double* mpStrainRateHost){ - checkMPMeshValid(p_mpmesh); - std::cerr << "Error: This routine is not implemented yet\n"; - exit(1); - (void)p_mpmesh; - (void)nComps; - (void)numMPs; - (void)mpStrainRateHost; -} -void polympo_setMPStress_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, const double* mpStressIn){ - checkMPMeshValid(p_mpmesh); - std::cerr << "Error: This routine is not implemented yet\n"; - exit(1); - (void)p_mpmesh; - (void)nComps; - (void)numMPs; - (void)mpStressIn; -} -void polympo_getMPStress_f(MPMesh_ptr p_mpmesh, const int nComps, const int numMPs, double* mpStressHost){ - checkMPMeshValid(p_mpmesh); - std::cerr << "Error: This routine is not implemented yet\n"; - exit(1); - (void)p_mpmesh; - (void)nComps; - (void)numMPs; - (void)mpStressHost; -} - -void polympo_startMeshFill_f(MPMesh_ptr p_mpmesh){ - checkMPMeshValid(p_mpmesh); - ((polyMPO::MPMesh*)p_mpmesh)->p_mesh->setMeshEdit(true); -} - -void polympo_endMeshFill_f(MPMesh_ptr p_mpmesh){ - checkMPMeshValid(p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - PMT_ALWAYS_ASSERT(p_mesh->meshEditable()); - p_mesh->setMeshEdit(false); -} - -void polympo_checkMeshMaxSettings_f(MPMesh_ptr p_mpmesh, const int maxEdges, const int vertexDegree){ - checkMPMeshValid(p_mpmesh); - PMT_ALWAYS_ASSERT(maxEdges <= maxVtxsPerElm); - PMT_ALWAYS_ASSERT(vertexDegree <= maxElmsPerVtx); -} - -void polympo_setMeshTypeGeneralPoly_f(MPMesh_ptr p_mpmesh){ - //chech validity - checkMPMeshValid(p_mpmesh); - ((polyMPO::MPMesh*)p_mpmesh)->p_mesh->setMeshType(polyMPO::mesh_general_polygonal); -} - -void polympo_setMeshTypeCVTPoly_f(MPMesh_ptr p_mpmesh){ - //chech validity - checkMPMeshValid(p_mpmesh); - ((polyMPO::MPMesh*)p_mpmesh)->p_mesh->setMeshType(polyMPO::mesh_CVT_polygonal); -} - -void polympo_setMeshGeomTypePlanar_f(MPMesh_ptr p_mpmesh){ - //chech validity - checkMPMeshValid(p_mpmesh); - ((polyMPO::MPMesh*)p_mpmesh)->p_mesh->setGeomType(polyMPO::geom_planar_surf); -} - -void polympo_setMeshGeomTypeSpherical_f(MPMesh_ptr p_mpmesh){ - //chech validity - checkMPMeshValid(p_mpmesh); - ((polyMPO::MPMesh*)p_mpmesh)->p_mesh->setGeomType(polyMPO::geom_spherical_surf); -} - -void polympo_setMeshSphereRadius_f(MPMesh_ptr p_mpmesh, const double sphereRadius){ - //chech validity - checkMPMeshValid(p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - PMT_ALWAYS_ASSERT(sphereRadius >= 0); - p_mesh->setSphereRadius(sphereRadius); -} - -void polympo_setMeshNumVtxs_f(MPMesh_ptr p_mpmesh, const int numVtxs){ - checkMPMeshValid(p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - p_mesh->setNumVtxs(numVtxs); - p_mesh->setMeshVtxBasedFieldSize(); -} - -void polympo_getMeshNumVtxs_f(MPMesh_ptr p_mpmesh, int & numVtxs) { - checkMPMeshValid(p_mpmesh); //chech vailidity - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - numVtxs = p_mesh->getNumVertices(); -} - -void polympo_setMeshNumElms_f(MPMesh_ptr p_mpmesh, const int numElms){ - checkMPMeshValid(p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - - auto elm2Vtx = polyMPO::IntVtx2ElmView("MeshElementsToVertices",numElms); - auto elm2Elm = polyMPO::IntElm2ElmView("MeshElementsToElements",numElms); - - p_mesh->setNumElms(numElms); - p_mesh->setElm2VtxConn(elm2Vtx); - p_mesh->setElm2ElmConn(elm2Elm); - p_mesh->setMeshElmBasedFieldSize(); -} - -void polympo_getMeshNumElms_f(MPMesh_ptr p_mpmesh, int & numElms) { - checkMPMeshValid(p_mpmesh); //chech vailidity - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - numElms = p_mesh->getNumElements(); -} - -void polympo_setMeshNumEdgesPerElm_f(MPMesh_ptr p_mpmesh, const int nCells, const int* array){ - //chech vailidity - checkMPMeshValid(p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - PMT_ALWAYS_ASSERT(p_mesh->meshEditable()); - - //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumElements()==nCells); - auto nEdgesPerElm = create_mirror_view_and_copy(array, nCells); - auto elm2VtxConn = p_mesh->getElm2VtxConn(); - auto elm2ElmConn = p_mesh->getElm2ElmConn(); - Kokkos::parallel_for("set nEdgesPerElm", nCells, KOKKOS_LAMBDA(const int elm){ - elm2VtxConn(elm,0) = nEdgesPerElm(elm); - elm2ElmConn(elm,0) = nEdgesPerElm(elm); - }); -} - -void polympo_setMeshElm2VtxConn_f(MPMesh_ptr p_mpmesh, const int maxEdges, const int nCells, const int* array){ - //chech vailidity - checkMPMeshValid(p_mpmesh); - kkViewHostU arrayHost(array,maxEdges,nCells); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - PMT_ALWAYS_ASSERT(p_mesh->meshEditable()); - - //check the size - PMT_ALWAYS_ASSERT(maxEdges <= maxVtxsPerElm); - PMT_ALWAYS_ASSERT(nCells == p_mesh->getNumElements()); - - Kokkos::View elm2VtxArray("MeshElementsToVertices",maxEdges,nCells); - Kokkos::deep_copy(elm2VtxArray, arrayHost); - auto elm2VtxConn = p_mesh->getElm2VtxConn(); - Kokkos::parallel_for("set elm2VtxConn", nCells, KOKKOS_LAMBDA(const int elm){ - for(int i=0; i arrayHost(array,maxEdges,nCells); //Fortran is column-major - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - PMT_ALWAYS_ASSERT(p_mesh->meshEditable()); - - //check the size - PMT_ALWAYS_ASSERT(maxEdges <= maxVtxsPerElm); - PMT_ALWAYS_ASSERT(nCells == p_mesh->getNumElements()); - - Kokkos::View elm2ElmArray("MeshElementsToVertices",maxEdges,nCells); - Kokkos::deep_copy(elm2ElmArray, arrayHost); - auto elm2ElmConn = p_mesh->getElm2ElmConn(); - Kokkos::parallel_for("set elm2ElmConn", nCells, KOKKOS_LAMBDA(const int elm){ - for(int i=0; ip_mesh; - - //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); - - //copy the host array to the device - auto coordsArray = p_mesh->getMeshField(); - auto h_coordsArray = Kokkos::create_mirror_view(coordsArray); - for(int i=0; ip_mesh; - - //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); - - //copy the device to host - auto coordsArray = p_mesh->getMeshField(); - auto h_coordsArray = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), - coordsArray); - for(int i=0; ip_mesh; - - //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); - - //copy the host array to the device - auto coordsArray = p_mesh->getMeshField(); - auto h_coordsArray = Kokkos::create_mirror_view(coordsArray); - for(int i=0; ip_mesh; - - //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); - - //copy the device to host - auto coordsArray = p_mesh->getMeshField(); - auto h_coordsArray = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), - coordsArray); - for(int i=0; ip_mesh; - - //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumElements()==nCells); - - //copy the host array to the device - auto elmCenter = p_mesh->getMeshField(); - auto h_elmCenter = Kokkos::create_mirror_view(elmCenter); - for(int i=0; ip_mesh; - - //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumElements()==nCells); - - //copy the device to host - auto elmCenter = p_mesh->getMeshField(); - auto h_elmCenter = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), elmCenter); - for(int i=0; ip_mesh; - - PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); - //copy the host array to the device - auto dualArea = p_mesh->getMeshField(); - auto h_dualArea = Kokkos::create_mirror_view(dualArea); - for(int i=0; ip_mesh; - - PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); - //copy the device to host - auto dualArea = p_mesh->getMeshField(); - auto h_dualArea = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), dualArea); - for(int i=0; ip_mesh; - - //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); - - //copy the host array to the device - auto coordsArray = p_mesh->getMeshField(); - auto h_coordsArray = Kokkos::create_mirror_view(coordsArray); - for(int i=0; ip_mesh; - - //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumVertices() == nVertices); - - //copy the device array to the host - auto coordsArray = p_mesh->getMeshField(); - auto h_coordsArray = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(),coordsArray); - for(int i=0; ip_mesh; - - //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumVertices()==nVertices); - - //copy the host array to the device - auto coordsArray = p_mesh->getMeshField(); - auto h_coordsArray = Kokkos::create_mirror_view(coordsArray); - for(int i=0; ip_mesh; - - //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumVertices() == nVertices); - - //copy the device array to the host - auto coordsArray = p_mesh->getMeshField(); - auto h_coordsArray = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(),coordsArray); - for(int i=0; ip_mesh; - - //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumElements()==nCells); - - //copy the host array to the device - auto coordsArray = p_mesh->getMeshField(); - auto h_coordsArray = Kokkos::create_mirror_view(coordsArray); - kkViewHostU arrayHost(elmMass, nCells); - for(int i=0; ip_mesh; - - //check the size - PMT_ALWAYS_ASSERT(p_mesh->getNumElements() == nCells); - - //copy the device array to the host - auto coordsArray = p_mesh->getMeshField(); - auto h_coordsArray = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(),coordsArray); - for(int i=0; ip_mesh; - kkViewHostU arrayHost(array,nComps,nVertices); - Kokkos::View array_d("meshVelIncrDevice",nComps,nVertices); - Kokkos::deep_copy(array_d, arrayHost); - - auto vtxField = p_mesh->getMeshField(); - - //check the size - PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); - PMT_ALWAYS_ASSERT(static_cast(nVertices*vec2d_nEntries)==vtxField.size()); - - //copy the host array to the device - Kokkos::parallel_for("set mesh dispIncr", nVertices, KOKKOS_LAMBDA(const int iVtx){ - vtxField(iVtx,0) = array_d(0,iVtx); - vtxField(iVtx,1) = array_d(1,iVtx); - }); - pumipic::RecordTime("PolyMPO_setMeshVtxOnSurfVelIncr", timer.seconds()); -} - -void polympo_setMeshVtxOnSurfDispIncr_f(MPMesh_ptr p_mpmesh, const int nComps, const int nVertices, const double* array) { - Kokkos::Timer timer; - //check mpMesh is valid - checkMPMeshValid(p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - kkViewHostU arrayHost(array,nComps,nVertices); - Kokkos::View array_d("meshDispIncrDevice",nComps,nVertices); - Kokkos::deep_copy(array_d, arrayHost); - - auto vtxField = p_mesh->getMeshField(); - - //check the size - PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); - PMT_ALWAYS_ASSERT(static_cast(nVertices*vec2d_nEntries)==vtxField.size()); - - //copy the host array to the device - Kokkos::parallel_for("set mesh dispIncr", nVertices, KOKKOS_LAMBDA(const int iVtx){ - vtxField(iVtx,0) = array_d(0,iVtx); - vtxField(iVtx,1) = array_d(1,iVtx); - }); - pumipic::RecordTime("PolyMPO_setMeshVtxOnSurfDispIncr", timer.seconds()); -} - - -void polympo_getMeshVtxOnSurfVeloIncr_f(MPMesh_ptr p_mpmesh, const int nComps, const int nVertices, double* array) { - //check mpMesh is valid - checkMPMeshValid(p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - kkDbl2dViewHostU arrayHost(array,nComps,nVertices); - Kokkos::View array_d("meshVelIncrDevice",nComps,nVertices); - - auto vtxField = p_mesh->getMeshField(); - - //check the size - PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); - PMT_ALWAYS_ASSERT(p_mesh->getNumVertices() == nVertices); - PMT_ALWAYS_ASSERT(static_cast(nVertices*vec2d_nEntries)==vtxField.size()); - - //copy the device array to the host - Kokkos::parallel_for("get mesh dispIncr", nVertices, KOKKOS_LAMBDA(const int iVtx){ - array_d(0,iVtx) = vtxField(iVtx,0); - array_d(1,iVtx) = vtxField(iVtx,1); - }); - Kokkos::deep_copy(arrayHost, array_d); -} - -void polympo_getMeshVtxOnSurfDispIncr_f(MPMesh_ptr p_mpmesh, const int nComps, const int nVertices, double* array) { - //check mpMesh is valid - checkMPMeshValid(p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - kkDbl2dViewHostU arrayHost(array,nComps,nVertices); - Kokkos::View array_d("meshDispIncrDevice",nComps,nVertices); - - auto vtxField = p_mesh->getMeshField(); - - //check the size - PMT_ALWAYS_ASSERT(nComps == vec2d_nEntries); - PMT_ALWAYS_ASSERT(p_mesh->getNumVertices() == nVertices); - PMT_ALWAYS_ASSERT(static_cast(nVertices*vec2d_nEntries)==vtxField.size()); - - //copy the device array to the host - Kokkos::parallel_for("get mesh dispIncr", nVertices, KOKKOS_LAMBDA(const int iVtx){ - array_d(0,iVtx) = vtxField(iVtx,0); - array_d(1,iVtx) = vtxField(iVtx,1); - }); - Kokkos::deep_copy(arrayHost, array_d); -} - -bool polympo_push1P_f(MPMesh_ptr p_mpmesh){ - checkMPMeshValid(p_mpmesh); - bool is_migrating=((polyMPO::MPMesh*)p_mpmesh)->push1P(); - return is_migrating; -} - -void polympo_push_ahead_f(MPMesh_ptr p_mpmesh){ - checkMPMeshValid(p_mpmesh); - ((polyMPO::MPMesh*)p_mpmesh)->push_ahead(); -} - -void polympo_push_swap_f(MPMesh_ptr p_mpmesh){ - checkMPMeshValid(p_mpmesh); - ((polyMPO::MPMesh*)p_mpmesh)->push_swap(); -} - -void polympo_push_swap_pos_f(MPMesh_ptr p_mpmesh){ - checkMPMeshValid(p_mpmesh); - ((polyMPO::MPMesh*)p_mpmesh)->push_swap_pos(); -} - -void polympo_push_f(MPMesh_ptr p_mpmesh){ - checkMPMeshValid(p_mpmesh); - ((polyMPO::MPMesh*)p_mpmesh)->push(); -} - -//TODO skeleton of reconstruction functions -void polympo_setReconstructionOfMass_f(MPMesh_ptr p_mpmesh, const int order, const int meshEntType){ - checkMPMeshValid(p_mpmesh); - auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); - polyMPO::MeshFieldType type = static_cast(meshEntType); - if (type == polyMPO::MeshFType_VtxBased) - mpmesh->setReconstructSlice(order, type); - if (type == polyMPO::MeshFType_ElmBased) - mpmesh->setReconstructSlice(order, type); -} - -void polympo_setReconstructionOfVel_f(MPMesh_ptr p_mpmesh, const int order, const int meshEntType){ - checkMPMeshValid(p_mpmesh); - auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); - polyMPO::MeshFieldType type = static_cast(meshEntType); - if (type == polyMPO::MeshFType_VtxBased) - mpmesh->setReconstructSlice(order, type); - else { - std::cerr << "Error: This reconstruction is not supported\n"; - exit(1); - } -} - -void polympo_setReconstructionOfStrainRate_f(MPMesh_ptr p_mpmesh, const int order, const int meshEntType){ - checkMPMeshValid(p_mpmesh); - std::cerr << "Error: This routine is not implemented yet\n"; - exit(1); - (void)order; - (void)meshEntType; -} - -void polympo_setReconstructionOfStress_f(MPMesh_ptr p_mpmesh, const int order, const int meshEntType){ - checkMPMeshValid(p_mpmesh); - std::cerr << "Error: This routine is not implemented yet\n"; - exit(1); - (void)order; - (void)meshEntType; -} - -//With MPI communication done via MPAS -void polympo_vtxSubAssemblyIceArea_f(MPMesh_ptr p_mpmesh, int vtxPerElm, int nCellsPlus1, int comp, double* array){ - checkMPMeshValid(p_mpmesh); - auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - PMT_ALWAYS_ASSERT(vtxPerElm <= maxVtxsPerElm); - PMT_ALWAYS_ASSERT(nCellsPlus1 == p_mesh->getNumElements()+1); - PMT_ALWAYS_ASSERT(comp == 0 || comp== 1); //either first or second component - mpmesh->subAssemblyVtx1(vtxPerElm, nCellsPlus1, comp, array); -} - -void polympo_vtxSubAssemblyVelocity_f(MPMesh_ptr p_mpmesh, int vtxPerElm, int nCellsPlus1, int comp, double* array){ - checkMPMeshValid(p_mpmesh); - auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - PMT_ALWAYS_ASSERT(vtxPerElm <= maxVtxsPerElm); - PMT_ALWAYS_ASSERT(nCellsPlus1 == p_mesh->getNumElements()+1); - PMT_ALWAYS_ASSERT(comp == 0 || comp== 1); //either first or second component - mpmesh->subAssemblyVtx1(vtxPerElm, nCellsPlus1, comp, array); -} - -void polympo_subAssemblyCoeffs_f(MPMesh_ptr p_mpmesh, int vtxPerElm, int nCellsPlus1, double* m11, double* m12, double* m13, double* m14, - double* m22, double* m23, double* m24, - double* m33, double* m34, - double* m44){ - checkMPMeshValid(p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - PMT_ALWAYS_ASSERT(vtxPerElm <= maxVtxsPerElm); - PMT_ALWAYS_ASSERT(nCellsPlus1 == p_mesh->getNumElements()+1); - auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); - mpmesh->subAssemblyCoeffs(vtxPerElm, nCellsPlus1, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44); -} - -void polympo_regularize_and_solve_matrix_f(MPMesh_ptr p_mpmesh, int nVerticesPlus1, double* m11, double* m12, double* m13, double* m14, - double* m22, double* m23, double* m24, - double* m33, double* m34, - double* m44){ - checkMPMeshValid(p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - PMT_ALWAYS_ASSERT(nVerticesPlus1 == p_mesh->getNumVertices()+1); - auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); - mpmesh->solveMatrixAndRegularize(nVerticesPlus1, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44); -} - -void polympo_applyReconstruction_f(MPMesh_ptr p_mpmesh){ - checkMPMeshValid(p_mpmesh); - auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); - mpmesh->reconstructSlices(); -} - -void polympo_setOwningProc_f(MPMesh_ptr p_mpmesh, const int nCells, const int* array){ - checkMPMeshValid(p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - PMT_ALWAYS_ASSERT(p_mesh->meshEditable()); - kkViewHostU arrayHost(array,nCells); - - //check the size - PMT_ALWAYS_ASSERT(nCells == p_mesh->getNumElements()); - - Kokkos::View owningProc("owningProc",nCells); - Kokkos::deep_copy(owningProc, arrayHost); - p_mesh->setOwningProc(owningProc); -} - -void polympo_setElmGlobal_f(MPMesh_ptr p_mpmesh, const int nCells, const int* array){ - checkMPMeshValid(p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - Kokkos::View arrayHost("arrayHost", nCells); - for (int i = 0; i < nCells; i++) { - arrayHost(i) = array[i] - 1; // TODO right now elmID offset is set after MPs initialized - } - //check the size - PMT_ALWAYS_ASSERT(nCells == p_mesh->getNumElements()); - - Kokkos::View elmGlobal("elmGlobal",nCells); - Kokkos::deep_copy(elmGlobal, arrayHost); - p_mesh->setElmGlobal(elmGlobal); -} - -void polympo_setVtxGlobal_f(MPMesh_ptr p_mpmesh, const int nVertices, const int* array){ - checkMPMeshValid(p_mpmesh); - auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - PMT_ALWAYS_ASSERT(nVertices==p_mesh->getNumVertices()); - Kokkos::View arrayHost("arrayHost", nVertices); - for (int i = 0; i < nVertices; i++) { - arrayHost(i) = array[i] - 1; // TODO right now elmID offset is set after MPs initialized - } - - Kokkos::View vtxGlobal("vtxGlobal",nVertices); - Kokkos::deep_copy(vtxGlobal, arrayHost); - p_mesh->setVtxGlobal(vtxGlobal); -} - -int polympo_getMPCount_f(MPMesh_ptr p_mpmesh) { - auto p_MPs = ((polyMPO::MPMesh*)p_mpmesh)->p_MPs; - return p_MPs->getCount(); -} - -void polympo_enableTiming_f(){ - pumipic::EnableTiming(); -} - -void polympo_summarizeTime_f(){ - pumipic::SummarizeTime(); -} - -void polympo_setTimingVerbosity_f(int v){ - pumipic::SetTimingVerbosity(v); -} - From 8a92ee9f08f9372e9ae4ab6eb57a90b2f04d2242 Mon Sep 17 00:00:00 2001 From: Nath Date: Wed, 13 Aug 2025 19:06:38 -0400 Subject: [PATCH 41/42] PR changes2_v2 --- src/pmpo_c.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index eea7dec..09ff941 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -61,7 +61,7 @@ void polympo_setMPICommunicator_f(MPMesh_ptr p_mpmesh, MPI_Fint fcomm){ void polympo_createMPs_f(MPMesh_ptr p_mpmesh, const int numElms, - const int numMPs, // total number of MPs which is >= no of active MPs + const int numMPs, // total number of MPs which is >= number of active MPs int* mpsPerElm, const int* mp2Elm, const int* isMPActive) { From fe854411113b8c4c451efba19134cbf93518f28a Mon Sep 17 00:00:00 2001 From: Nath Date: Thu, 14 Aug 2025 13:41:21 -0400 Subject: [PATCH 42/42] PR changes3_v0 --- src/pmpo_MPMesh.hpp | 6 +-- src/pmpo_MPMesh_assembly.hpp | 94 ++++++++++++++++++------------------ src/pmpo_c.cpp | 26 +++++----- src/pmpo_c.h | 8 +-- src/pmpo_fortran.f90 | 16 +++--- 5 files changed, 74 insertions(+), 76 deletions(-) diff --git a/src/pmpo_MPMesh.hpp b/src/pmpo_MPMesh.hpp index 36e9056..cb5c893 100644 --- a/src/pmpo_MPMesh.hpp +++ b/src/pmpo_MPMesh.hpp @@ -66,13 +66,13 @@ class MPMesh{ template void assemblyVtx1(); template - void subAssemblyVtx1(int vtxPerElm, int nCellsPlus1, int comp, double* array); + void subAssemblyVtx1(int vtxPerElm, int nCells, int comp, double* array); - void subAssemblyCoeffs(int vtxPerElm, int nCellsPlus1, double* m11, double* m12, double* m13, double* m14, + void subAssemblyCoeffs(int vtxPerElm, int nCells, double* m11, double* m12, double* m13, double* m14, double* m22, double* m23, double* m24, double* m33, double* m34, double* m44); - void solveMatrixAndRegularize(int nVerticesPlus1, double* m11, double* m12, double* m13, double* m14, + void solveMatrixAndRegularize(int nVertices, double* m11, double* m12, double* m13, double* m14, double* m22, double* m23, double* m24, double* m33, double* m34, double* m44); diff --git a/src/pmpo_MPMesh_assembly.hpp b/src/pmpo_MPMesh_assembly.hpp index ee26abc..033386e 100644 --- a/src/pmpo_MPMesh_assembly.hpp +++ b/src/pmpo_MPMesh_assembly.hpp @@ -210,7 +210,7 @@ void MPMesh::computeMatricesAndSolve(){ } //Method 2 for coefficients -void MPMesh::subAssemblyCoeffs(int vtxPerElm, int nCellsPlus1, double* m11, double* m12, double* m13, double* m14, +void MPMesh::subAssemblyCoeffs(int vtxPerElm, int nCells, double* m11, double* m12, double* m13, double* m14, double* m22, double* m23, double* m24, double* m33, double* m34, double* m44){ @@ -242,16 +242,16 @@ void MPMesh::subAssemblyCoeffs(int vtxPerElm, int nCellsPlus1, double* m11, doub if(p_mesh->getGeomType() == geom_spherical_surf) radius=p_mesh->getSphereRadius(); - Kokkos::View m11_d("m11", vtxPerElm, nCellsPlus1); - Kokkos::View m12_d("m12", vtxPerElm, nCellsPlus1); - Kokkos::View m13_d("m13", vtxPerElm, nCellsPlus1); - Kokkos::View m14_d("m14", vtxPerElm, nCellsPlus1); - Kokkos::View m22_d("m22", vtxPerElm, nCellsPlus1); - Kokkos::View m23_d("m23", vtxPerElm, nCellsPlus1); - Kokkos::View m24_d("m23", vtxPerElm, nCellsPlus1); - Kokkos::View m33_d("m33", vtxPerElm, nCellsPlus1); - Kokkos::View m34_d("m34", vtxPerElm, nCellsPlus1); - Kokkos::View m44_d("m34", vtxPerElm, nCellsPlus1); + Kokkos::View m11_d("m11", vtxPerElm, nCells); + Kokkos::View m12_d("m12", vtxPerElm, nCells); + Kokkos::View m13_d("m13", vtxPerElm, nCells); + Kokkos::View m14_d("m14", vtxPerElm, nCells); + Kokkos::View m22_d("m22", vtxPerElm, nCells); + Kokkos::View m23_d("m23", vtxPerElm, nCells); + Kokkos::View m24_d("m23", vtxPerElm, nCells); + Kokkos::View m33_d("m33", vtxPerElm, nCells); + Kokkos::View m34_d("m34", vtxPerElm, nCells); + Kokkos::View m44_d("m34", vtxPerElm, nCells); auto sub_assemble = PS_LAMBDA(const int& elm, const int& mp, const int& mask) { if(mask && (elm2Process(elm)==comm_rank)) { //if material point is 'active'/'enabled' @@ -279,16 +279,16 @@ void MPMesh::subAssemblyCoeffs(int vtxPerElm, int nCellsPlus1, double* m11, doub pumipic::RecordTime("VtxSubAssemblyComputeCoeff", timer.seconds()); Kokkos::Timer timer2; - kkDbl2dViewHostU m11_h(m11, vtxPerElm, nCellsPlus1); - kkDbl2dViewHostU m12_h(m12, vtxPerElm, nCellsPlus1); - kkDbl2dViewHostU m13_h(m13, vtxPerElm, nCellsPlus1); - kkDbl2dViewHostU m14_h(m14, vtxPerElm, nCellsPlus1); - kkDbl2dViewHostU m22_h(m22, vtxPerElm, nCellsPlus1); - kkDbl2dViewHostU m23_h(m23, vtxPerElm, nCellsPlus1); - kkDbl2dViewHostU m24_h(m24, vtxPerElm, nCellsPlus1); - kkDbl2dViewHostU m33_h(m33, vtxPerElm, nCellsPlus1); - kkDbl2dViewHostU m34_h(m34, vtxPerElm, nCellsPlus1); - kkDbl2dViewHostU m44_h(m44, vtxPerElm, nCellsPlus1); + kkDbl2dViewHostU m11_h(m11, vtxPerElm, nCells); + kkDbl2dViewHostU m12_h(m12, vtxPerElm, nCells); + kkDbl2dViewHostU m13_h(m13, vtxPerElm, nCells); + kkDbl2dViewHostU m14_h(m14, vtxPerElm, nCells); + kkDbl2dViewHostU m22_h(m22, vtxPerElm, nCells); + kkDbl2dViewHostU m23_h(m23, vtxPerElm, nCells); + kkDbl2dViewHostU m24_h(m24, vtxPerElm, nCells); + kkDbl2dViewHostU m33_h(m33, vtxPerElm, nCells); + kkDbl2dViewHostU m34_h(m34, vtxPerElm, nCells); + kkDbl2dViewHostU m44_h(m44, vtxPerElm, nCells); Kokkos::deep_copy(m11_h, m11_d); Kokkos::deep_copy(m12_h, m12_d); @@ -305,33 +305,33 @@ void MPMesh::subAssemblyCoeffs(int vtxPerElm, int nCellsPlus1, double* m11, doub } //Method 2 for coefficients Solve matrix -void MPMesh::solveMatrixAndRegularize(int nVerticesPlus1, double* m11, double* m12, double* m13, double* m14, +void MPMesh::solveMatrixAndRegularize(int nVertices, double* m11, double* m12, double* m13, double* m14, double* m22, double* m23, double* m24, double* m33, double* m34, double* m44){ Kokkos::Timer timer; - kkViewHostU m11_h(m11, nVerticesPlus1); - kkViewHostU m12_h(m12, nVerticesPlus1); - kkViewHostU m13_h(m13, nVerticesPlus1); - kkViewHostU m14_h(m14, nVerticesPlus1); - kkViewHostU m22_h(m22, nVerticesPlus1); - kkViewHostU m23_h(m23, nVerticesPlus1); - kkViewHostU m24_h(m24, nVerticesPlus1); - kkViewHostU m33_h(m33, nVerticesPlus1); - kkViewHostU m34_h(m34, nVerticesPlus1); - kkViewHostU m44_h(m44, nVerticesPlus1); + kkViewHostU m11_h(m11, nVertices); + kkViewHostU m12_h(m12, nVertices); + kkViewHostU m13_h(m13, nVertices); + kkViewHostU m14_h(m14, nVertices); + kkViewHostU m22_h(m22, nVertices); + kkViewHostU m23_h(m23, nVertices); + kkViewHostU m24_h(m24, nVertices); + kkViewHostU m33_h(m33, nVertices); + kkViewHostU m34_h(m34, nVertices); + kkViewHostU m44_h(m44, nVertices); - Kokkos::View m11_d("m11", nVerticesPlus1); - Kokkos::View m12_d("m12", nVerticesPlus1); - Kokkos::View m13_d("m13", nVerticesPlus1); - Kokkos::View m14_d("m14", nVerticesPlus1); - Kokkos::View m22_d("m22", nVerticesPlus1); - Kokkos::View m23_d("m23", nVerticesPlus1); - Kokkos::View m24_d("m24", nVerticesPlus1); - Kokkos::View m33_d("m33", nVerticesPlus1); - Kokkos::View m34_d("m34", nVerticesPlus1); - Kokkos::View m44_d("m44", nVerticesPlus1); + Kokkos::View m11_d("m11", nVertices); + Kokkos::View m12_d("m12", nVertices); + Kokkos::View m13_d("m13", nVertices); + Kokkos::View m14_d("m14", nVertices); + Kokkos::View m22_d("m22", nVertices); + Kokkos::View m23_d("m23", nVertices); + Kokkos::View m24_d("m24", nVertices); + Kokkos::View m33_d("m33", nVertices); + Kokkos::View m34_d("m34", nVertices); + Kokkos::View m44_d("m44", nVertices); Kokkos::deep_copy(m11_d, m11_h); Kokkos::deep_copy(m12_d, m12_h); @@ -347,9 +347,9 @@ void MPMesh::solveMatrixAndRegularize(int nVerticesPlus1, double* m11, double* m Kokkos::Timer timer2; auto dual_triangle_area=p_mesh->getMeshField(); - Kokkos::View VtxCoeffs("VtxCoeffs", nVerticesPlus1); + Kokkos::View VtxCoeffs("VtxCoeffs", nVertices); double radius=p_mesh->getSphereRadius(); - Kokkos::parallel_for("fill", nVerticesPlus1, KOKKOS_LAMBDA(const int vtx){ + Kokkos::parallel_for("fill", nVertices, KOKKOS_LAMBDA(const int vtx){ Vec4d v0 = {m11_d(vtx), m12_d(vtx), m13_d(vtx), m14_d(vtx)}; Vec4d v1 = {m12_d(vtx), m22_d(vtx), m23_d(vtx), m24_d(vtx)}; Vec4d v2 = {m13_d(vtx), m23_d(vtx), m33_d(vtx), m34_d(vtx)}; @@ -378,7 +378,7 @@ void MPMesh::solveMatrixAndRegularize(int nVerticesPlus1, double* m11, double* m //Method2 template -void MPMesh::subAssemblyVtx1(int vtxPerElm, int nCellsPlus1, int comp, double* array) { +void MPMesh::subAssemblyVtx1(int vtxPerElm, int nCells, int comp, double* array) { Kokkos::Timer timer; auto VtxCoeffs=this->precomputedVtxCoeffs; @@ -402,7 +402,7 @@ void MPMesh::subAssemblyVtx1(int vtxPerElm, int nCellsPlus1, int comp, double* a double radius=p_mesh->getSphereRadius(); - Kokkos::View array_d("reconstructedIceArea", vtxPerElm, nCellsPlus1); + Kokkos::View array_d("reconstructedIceArea", vtxPerElm, nCells); auto sub_assemble = PS_LAMBDA(const int& elm, const int& mp, const int& mask) { if(mask && (elm2Process(elm)==comm_rank)) { int nVtxE = elm2VtxConn(elm,0); //number of vertices bounding the element @@ -426,7 +426,7 @@ void MPMesh::subAssemblyVtx1(int vtxPerElm, int nCellsPlus1, int comp, double* a pumipic::RecordTime("polyMPOsubAssemblyFieldCompute", timer.seconds()); Kokkos::Timer timer2; - kkDbl2dViewHostU arrayHost(array, vtxPerElm, nCellsPlus1); + kkDbl2dViewHostU arrayHost(array, vtxPerElm, nCells); Kokkos::deep_copy(arrayHost, array_d); pumipic::RecordTime("PolyMPOsubAssemblyFieldGet", timer2.seconds()); } diff --git a/src/pmpo_c.cpp b/src/pmpo_c.cpp index 09ff941..2369d57 100644 --- a/src/pmpo_c.cpp +++ b/src/pmpo_c.cpp @@ -99,8 +99,6 @@ void polympo_createMPs_f(MPMesh_ptr p_mpmesh, int globalFirstElmWithMPs; MPI_Allreduce(&firstElmWithMPs, &globalFirstElmWithMPs, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD); - //printf("With a MP, globally smallest mesh elm %d and first elm %d \n", globalMinElmID, globalFirstElmWithMPs); - int offset = -1; if(globalMinElmID-globalFirstElmWithMPs==1) { offset = 1; @@ -1239,47 +1237,47 @@ void polympo_setReconstructionOfStress_f(MPMesh_ptr p_mpmesh, const int order, c } //With MPI communication done via MPAS -void polympo_vtxSubAssemblyIceArea_f(MPMesh_ptr p_mpmesh, int vtxPerElm, int nCellsPlus1, int comp, double* array){ +void polympo_vtxSubAssemblyIceArea_f(MPMesh_ptr p_mpmesh, int vtxPerElm, int nCells, int comp, double* array){ checkMPMeshValid(p_mpmesh); auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; PMT_ALWAYS_ASSERT(vtxPerElm <= maxVtxsPerElm); - PMT_ALWAYS_ASSERT(nCellsPlus1 == p_mesh->getNumElements()+1); + PMT_ALWAYS_ASSERT(nCells == p_mesh->getNumElements()); PMT_ALWAYS_ASSERT(comp == 0 || comp== 1); //either first or second component - mpmesh->subAssemblyVtx1(vtxPerElm, nCellsPlus1, comp, array); + mpmesh->subAssemblyVtx1(vtxPerElm, nCells, comp, array); } -void polympo_vtxSubAssemblyVelocity_f(MPMesh_ptr p_mpmesh, int vtxPerElm, int nCellsPlus1, int comp, double* array){ +void polympo_vtxSubAssemblyVelocity_f(MPMesh_ptr p_mpmesh, int vtxPerElm, int nCells, int comp, double* array){ checkMPMeshValid(p_mpmesh); auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; PMT_ALWAYS_ASSERT(vtxPerElm <= maxVtxsPerElm); - PMT_ALWAYS_ASSERT(nCellsPlus1 == p_mesh->getNumElements()+1); + PMT_ALWAYS_ASSERT(nCells == p_mesh->getNumElements()); PMT_ALWAYS_ASSERT(comp == 0 || comp== 1); //either first or second component - mpmesh->subAssemblyVtx1(vtxPerElm, nCellsPlus1, comp, array); + mpmesh->subAssemblyVtx1(vtxPerElm, nCells, comp, array); } -void polympo_subAssemblyCoeffs_f(MPMesh_ptr p_mpmesh, int vtxPerElm, int nCellsPlus1, double* m11, double* m12, double* m13, double* m14, +void polympo_subAssemblyCoeffs_f(MPMesh_ptr p_mpmesh, int vtxPerElm, int nCells, double* m11, double* m12, double* m13, double* m14, double* m22, double* m23, double* m24, double* m33, double* m34, double* m44){ checkMPMeshValid(p_mpmesh); auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; PMT_ALWAYS_ASSERT(vtxPerElm <= maxVtxsPerElm); - PMT_ALWAYS_ASSERT(nCellsPlus1 == p_mesh->getNumElements()+1); + PMT_ALWAYS_ASSERT(nCells == p_mesh->getNumElements()); auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); - mpmesh->subAssemblyCoeffs(vtxPerElm, nCellsPlus1, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44); + mpmesh->subAssemblyCoeffs(vtxPerElm, nCells, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44); } -void polympo_regularize_and_solve_matrix_f(MPMesh_ptr p_mpmesh, int nVerticesPlus1, double* m11, double* m12, double* m13, double* m14, +void polympo_regularize_and_solve_matrix_f(MPMesh_ptr p_mpmesh, int nVertices, double* m11, double* m12, double* m13, double* m14, double* m22, double* m23, double* m24, double* m33, double* m34, double* m44){ checkMPMeshValid(p_mpmesh); auto p_mesh = ((polyMPO::MPMesh*)p_mpmesh)->p_mesh; - PMT_ALWAYS_ASSERT(nVerticesPlus1 == p_mesh->getNumVertices()+1); + PMT_ALWAYS_ASSERT(nVertices == p_mesh->getNumVertices()); auto mpmesh = ((polyMPO::MPMesh*)p_mpmesh); - mpmesh->solveMatrixAndRegularize(nVerticesPlus1, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44); + mpmesh->solveMatrixAndRegularize(nVertices, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44); } void polympo_applyReconstruction_f(MPMesh_ptr p_mpmesh){ diff --git a/src/pmpo_c.h b/src/pmpo_c.h index 1f066f5..d39c195 100644 --- a/src/pmpo_c.h +++ b/src/pmpo_c.h @@ -113,13 +113,13 @@ void polympo_setReconstructionOfStress_f(MPMesh_ptr p_mpmesh, const int order, c void polympo_applyReconstruction_f(MPMesh_ptr p_mpmesh); //Reconstruction using MPAS -void polympo_vtxSubAssemblyIceArea_f(MPMesh_ptr p_mpmesh, int vtxPerElm, int nCellsPlus1, int comp, double* array); -void polympo_vtxSubAssemblyVelocity_f(MPMesh_ptr p_mpmesh, int vtxPerElm, int nCellsPlus1, int comp, double* array); -void polympo_subAssemblyCoeffs_f(MPMesh_ptr p_mpmesh, int vtxPerElm, int nCellsPlus1, double* m11, double* m12, double* m13, double* m14, +void polympo_vtxSubAssemblyIceArea_f(MPMesh_ptr p_mpmesh, int vtxPerElm, int nCells, int comp, double* array); +void polympo_vtxSubAssemblyVelocity_f(MPMesh_ptr p_mpmesh, int vtxPerElm, int nCells, int comp, double* array); +void polympo_subAssemblyCoeffs_f(MPMesh_ptr p_mpmesh, int vtxPerElm, int nCells, double* m11, double* m12, double* m13, double* m14, double* m22, double* m23, double* m24, double* m33, double* m34, double* m44); -void polympo_regularize_and_solve_matrix_f(MPMesh_ptr p_mpmesh, int nVeticesPlus1, double* m11, double* m12, double* m13, double* m14, +void polympo_regularize_and_solve_matrix_f(MPMesh_ptr p_mpmesh, int nVetices, double* m11, double* m12, double* m13, double* m14, double* m22, double* m23, double* m24, double* m33, double* m34, double* m44); diff --git a/src/pmpo_fortran.f90 b/src/pmpo_fortran.f90 index c5ef73e..10d04f2 100644 --- a/src/pmpo_fortran.f90 +++ b/src/pmpo_fortran.f90 @@ -910,35 +910,35 @@ subroutine polympo_setReconstructionOfStress(mpMesh, order, meshEntType) & end subroutine - subroutine polympo_vtxSubAssemblyIceArea(mpMesh, vtxPerElm, nCellsPlus1, comp, array) & + subroutine polympo_vtxSubAssemblyIceArea(mpMesh, vtxPerElm, nCells, comp, array) & bind(C, NAME='polympo_vtxSubAssemblyIceArea_f') use :: iso_c_binding type(c_ptr), value :: mpMesh - integer(c_int), value :: vtxPerElm, nCellsPlus1, comp + integer(c_int), value :: vtxPerElm, nCells, comp type(c_ptr), value :: array end subroutine - subroutine polympo_vtxSubAssemblyVelocity(mpMesh, vtxPerElm, nCellsPlus1, comp, array) & + subroutine polympo_vtxSubAssemblyVelocity(mpMesh, vtxPerElm, nCells, comp, array) & bind(C, NAME='polympo_vtxSubAssemblyVelocity_f') use :: iso_c_binding type(c_ptr), value :: mpMesh - integer(c_int), value :: vtxPerElm, nCellsPlus1, comp + integer(c_int), value :: vtxPerElm, nCells, comp type(c_ptr), value :: array end subroutine - subroutine polympo_subAssemblyCoeffs(mpMesh, vtxPerElm, nCellsPlus1, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44) & + subroutine polympo_subAssemblyCoeffs(mpMesh, vtxPerElm, nCells, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44) & bind(C, NAME='polympo_subAssemblyCoeffs_f') use :: iso_c_binding type(c_ptr), value :: mpMesh - integer(c_int), value :: vtxPerElm, nCellsPlus1 + integer(c_int), value :: vtxPerElm, nCells type(c_ptr), value :: m11, m12, m13, m14, m22, m23, m24, m33, m34, m44 end subroutine - subroutine polympo_regularize_and_solve_matrix(mpMesh, nVerticesPlus1, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44) & + subroutine polympo_regularize_and_solve_matrix(mpMesh, nVertices, m11, m12, m13, m14, m22, m23, m24, m33, m34, m44) & bind(C, NAME='polympo_regularize_and_solve_matrix_f') use :: iso_c_binding type(c_ptr), value :: mpMesh - integer(c_int), value :: nVerticesPlus1 + integer(c_int), value :: nVertices type(c_ptr), value :: m11, m12, m13, m14, m22, m23, m24, m33, m34, m44 end subroutine