diff --git a/lib/actions/ferm/invert/quda_solvers/syssolver_linop_nef_quda_w.cc b/lib/actions/ferm/invert/quda_solvers/syssolver_linop_nef_quda_w.cc index 0a0b7daf7..60deb8039 100644 --- a/lib/actions/ferm/invert/quda_solvers/syssolver_linop_nef_quda_w.cc +++ b/lib/actions/ferm/invert/quda_solvers/syssolver_linop_nef_quda_w.cc @@ -1,6 +1,6 @@ /*! \file -* \brief Solve a MdagM*psi=chi linear system by CG2 -*/ + * \brief Solve a MdagM*psi=chi linear system by CG2 + */ #include "actions/ferm/invert/syssolver_linop_factory.h" #include "actions/ferm/invert/syssolver_linop_aggregate.h" @@ -32,11 +32,11 @@ namespace Chroma bool registered = false; } - LinOpSystemSolverArray* createFerm(XMLReader& xml_in, - const std::string& path, - Handle< FermState< LatticeFermion, multi1d, multi1d > > state, - - Handle< LinearOperatorArray > A) + LinOpSystemSolverArray* createFerm(XMLReader& xml_in, + const std::string& path, + Handle< FermState< LatticeFermion, multi1d, multi1d > > state, + + Handle< LinearOperatorArray > A) { return new LinOpSysSolverQUDANEF(A, state,SysSolverQUDANEFParams(xml_in, path)); } @@ -46,10 +46,10 @@ namespace Chroma { bool success = true; if (! registered) - { - success &= Chroma::TheLinOpFermSystemSolverArrayFactory::Instance().registerObject(name, createFerm); - registered = true; - } + { + success &= Chroma::TheLinOpFermSystemSolverArrayFactory::Instance().registerObject(name, createFerm); + registered = true; + } return success; } } @@ -76,8 +76,8 @@ namespace Chroma #else //not yet //for(unsigned int s=0; s > A_, - Handle< FermState > state_, - const SysSolverQUDANEFParams& invParam_) : + Handle< FermState > state_, + const SysSolverQUDANEFParams& invParam_) : A(A_), invParam(invParam_) { START_CODE(); @@ -82,44 +82,44 @@ namespace Chroma int s = sizeof( WordType::Type_t ); if (s == 4) { - cpu_prec = QUDA_SINGLE_PRECISION; + cpu_prec = QUDA_SINGLE_PRECISION; } else { - cpu_prec = QUDA_DOUBLE_PRECISION; + cpu_prec = QUDA_DOUBLE_PRECISION; } // Work out GPU precision switch( invParam.cudaPrecision ) { case HALF: - gpu_prec = QUDA_HALF_PRECISION; - break; + gpu_prec = QUDA_HALF_PRECISION; + break; case SINGLE: - gpu_prec = QUDA_SINGLE_PRECISION; - break; + gpu_prec = QUDA_SINGLE_PRECISION; + break; case DOUBLE: - gpu_prec = QUDA_DOUBLE_PRECISION; - break; + gpu_prec = QUDA_DOUBLE_PRECISION; + break; default: - gpu_prec = cpu_prec; - break; + gpu_prec = cpu_prec; + break; } // Work out GPU Sloppy precision // Default: No Sloppy switch( invParam.cudaSloppyPrecision ) { case HALF: - gpu_half_prec = QUDA_HALF_PRECISION; - break; + gpu_half_prec = QUDA_HALF_PRECISION; + break; case SINGLE: - gpu_half_prec = QUDA_SINGLE_PRECISION; - break; + gpu_half_prec = QUDA_SINGLE_PRECISION; + break; case DOUBLE: - gpu_half_prec = QUDA_DOUBLE_PRECISION; - break; + gpu_half_prec = QUDA_DOUBLE_PRECISION; + break; default: - gpu_half_prec = gpu_prec; - break; + gpu_half_prec = gpu_prec; + break; } // 2) pull 'new; GAUGE and Invert params @@ -149,10 +149,10 @@ namespace Chroma // This flag just tells QUDA that this is so, // so that QUDA can take care in the reconstruct if( invParam.AntiPeriodicT ) { - q_gauge_param.t_boundary = QUDA_ANTI_PERIODIC_T; + q_gauge_param.t_boundary = QUDA_ANTI_PERIODIC_T; } else { - q_gauge_param.t_boundary = QUDA_PERIODIC_T; + q_gauge_param.t_boundary = QUDA_PERIODIC_T; } // Set cpu_prec, cuda_prec, reconstruct and sloppy versions @@ -162,34 +162,34 @@ namespace Chroma switch( invParam.cudaReconstruct ) { case RECONS_NONE: - q_gauge_param.reconstruct = QUDA_RECONSTRUCT_NO; - break; + q_gauge_param.reconstruct = QUDA_RECONSTRUCT_NO; + break; case RECONS_8: - q_gauge_param.reconstruct = QUDA_RECONSTRUCT_8; - break; + q_gauge_param.reconstruct = QUDA_RECONSTRUCT_8; + break; case RECONS_12: - q_gauge_param.reconstruct = QUDA_RECONSTRUCT_12; - break; + q_gauge_param.reconstruct = QUDA_RECONSTRUCT_12; + break; default: - q_gauge_param.reconstruct = QUDA_RECONSTRUCT_12; - break; + q_gauge_param.reconstruct = QUDA_RECONSTRUCT_12; + break; }; q_gauge_param.cuda_prec_sloppy = gpu_half_prec; switch( invParam.cudaSloppyReconstruct ) { case RECONS_NONE: - q_gauge_param.reconstruct_sloppy = QUDA_RECONSTRUCT_NO; - break; + q_gauge_param.reconstruct_sloppy = QUDA_RECONSTRUCT_NO; + break; case RECONS_8: - q_gauge_param.reconstruct_sloppy = QUDA_RECONSTRUCT_8; - break; + q_gauge_param.reconstruct_sloppy = QUDA_RECONSTRUCT_8; + break; case RECONS_12: - q_gauge_param.reconstruct_sloppy = QUDA_RECONSTRUCT_12; - break; + q_gauge_param.reconstruct_sloppy = QUDA_RECONSTRUCT_12; + break; default: - q_gauge_param.reconstruct_sloppy = QUDA_RECONSTRUCT_12; - break; + q_gauge_param.reconstruct_sloppy = QUDA_RECONSTRUCT_12; + break; }; // Gauge fixing: @@ -200,21 +200,21 @@ namespace Chroma // Now downcast to single prec fields. for(int mu=0; mu < Nd; mu++) { - links_single[mu] = (state_->getLinks())[mu]; + links_single[mu] = (state_->getLinks())[mu]; } // GaugeFix if( invParam.axialGaugeP ) { - QDPIO::cout << "Fixing Temporal Gauge" << std::endl; - temporalGauge(links_single, GFixMat, Nd-1); - for(int mu=0; mu < Nd; mu++){ - links_single[mu] = GFixMat*(state_->getLinks())[mu]*adj(shift(GFixMat, FORWARD, mu)); - } - q_gauge_param.gauge_fix = QUDA_GAUGE_FIXED_YES; + QDPIO::cout << "Fixing Temporal Gauge" << std::endl; + temporalGauge(links_single, GFixMat, Nd-1); + for(int mu=0; mu < Nd; mu++){ + links_single[mu] = GFixMat*(state_->getLinks())[mu]*adj(shift(GFixMat, FORWARD, mu)); + } + q_gauge_param.gauge_fix = QUDA_GAUGE_FIXED_YES; } else { - // No GaugeFix - q_gauge_param.gauge_fix = QUDA_GAUGE_FIXED_NO; // No Gfix yet + // No GaugeFix + q_gauge_param.gauge_fix = QUDA_GAUGE_FIXED_NO; // No Gfix yet } // Don't support anisotorpy for Moebius @@ -226,21 +226,21 @@ namespace Chroma // Invert type: switch( invParam.solverType ) { case CG: - quda_inv_param.inv_type = QUDA_CG_INVERTER; - solver_string = "CG"; - break; + quda_inv_param.inv_type = QUDA_CG_INVERTER; + solver_string = "CG"; + break; case BICGSTAB: - QDPIO::cerr << "Solver BICGSTAB not supported for MDWF" << std::endl; - QDP_abort(1); - break; + QDPIO::cerr << "Solver BICGSTAB not supported for MDWF" << std::endl; + QDP_abort(1); + break; case GCR: - QDPIO::cerr << "Solver GCR not supported for MDWF" << std::endl; - QDP_abort(1); - break; + QDPIO::cerr << "Solver GCR not supported for MDWF" << std::endl; + QDP_abort(1); + break; default: - QDPIO::cerr << "Unknown Solver type" << std::endl; - QDP_abort(1); - break; + QDPIO::cerr << "Unknown Solver type" << std::endl; + QDP_abort(1); + break; } // Mass @@ -249,8 +249,8 @@ namespace Chroma quda_inv_param.Ls=invParam.NEFParams.N5; // Mike made these static so no need to alloc them. if ( invParam.NEFParams.N5 >= QUDA_MAX_DWF_LS ) { - QDPIO::cerr << "LS can be at most " << QUDA_MAX_DWF_LS << std::endl; - QDP_abort(1); + QDPIO::cerr << "LS can be at most " << QUDA_MAX_DWF_LS << std::endl; + QDP_abort(1); } // Copy b5 and c5 into static array @@ -258,11 +258,11 @@ namespace Chroma QDPIO::cout << "Ls from params: " << invParam.NEFParams.N5 << std::endl; QDPIO::cout << "Ls from quda: " << quda_inv_param.Ls << std::endl; for(int s = 0; s < quda_inv_param.Ls; s++){ - quda_inv_param.b_5[s] = toDouble(invParam.NEFParams.b5[s]); - quda_inv_param.c_5[s] = toDouble(invParam.NEFParams.c5[s]); - QDPIO::cout << " b5[" < max_face ) { - max_face = face_size[i]; - } + if ( face_size[i] > max_face ) { + max_face = face_size[i]; + } } q_gauge_param.ga_pad = max_face; @@ -352,13 +352,18 @@ namespace Chroma quda_inv_param.maxiter_precondition = 1000; quda_inv_param.verbosity_precondition = QUDA_SILENT; quda_inv_param.gcrNkrylov = 1; - - - if( invParam.verboseP ) { - quda_inv_param.verbosity = QUDA_VERBOSE; + + if( invParam.verboseP ) { + if ( invParam.debugP ) { + QDPIO::cout << "Enabling QUDA_DEBUG_VERBOSE with Verbose=true and Debug=true" << std::endl; + quda_inv_param.verbosity = QUDA_DEBUG_VERBOSE; + } + else { + quda_inv_param.verbosity = QUDA_VERBOSE; + } } - else { - quda_inv_param.verbosity = QUDA_SUMMARIZE; + else { + quda_inv_param.verbosity = QUDA_SUMMARIZE; } // Set up the links @@ -366,7 +371,7 @@ namespace Chroma #ifndef BUILD_QUDA_DEVIFACE_GAUGE for(int mu=0; mu < Nd; mu++) { - gauge[mu] = (void *)&(links_single[mu].elem(all.start()).elem().elem(0,0).real()); + gauge[mu] = (void *)&(links_single[mu].elem(all.start()).elem().elem(0,0).real()); } #else GetMemoryPtrGauge(gauge,links_single); @@ -416,77 +421,77 @@ namespace Chroma #if 0 // Test code.... { - const multi1d& latdims = Layout::subgridLattSize(); - int halfsize=latdims[0]*latdims[1]*latdims[2]*latdims[3]/2; - int fermsize=halfsize*Nc*Ns*2; - - // In1 is input to Chroma, Out1 is result - multi1d in1( this->size() ); - multi1d out1(this->size() ); - - // In2 is input to QUDA, Out2 is result - multi1d in2( this->size() ); - multi1d out2(this->size() ); - - - for(int s=0; s < this->size(); s++ ) { - gaussian(in1[s]); // Gaussian into in1 - in2[s] = in1[s]; // copy to in2 - - } - - for(int d=0; d < 2; d++) { - for(int s=0; s < this->size(); s++ ) { - out1[s]=zero; // zero both out1 and out2 - out2[s]=zero; - } - - if ( d==0 ) { - // Apply A to in2 - QDPIO::cout << "DOing Mat" << std::endl; - (*A)(out2, in2, PLUS); - } - else { - QDPIO::cout << "Doing MatDag" << std::endl; - (*A)(out2, in2, MINUS); - } - - // Copy in1 into QUDA - REAL* spinorIn = new REAL[quda_inv_param.Ls*fermsize]; - REAL* spinorOut = new REAL[quda_inv_param.Ls*fermsize]; - memset((spinorIn), 0, fermsize*quda_inv_param.Ls*sizeof(REAL)); - memset((spinorOut), 0, fermsize*quda_inv_param.Ls*sizeof(REAL)); - - for(unsigned int s=0; s(&(psi_s[s].elem(all.start()).elem(0).elem(0).real())),0,fermsize*2*sizeof(REAL)); - memcpy((&out1[s].elem(rb[1].start()).elem(0).elem(0).real()),(&spinorOut[fermsize*s]),fermsize*sizeof(REAL)); - } - - // Now compare out1 and out2 - for(int s=0; s < this->size();s++) { - out1[s] *= invTwoKappaB; - - QDPIO::cout << "s=" << s << " diff=" << norm2(out2[s]-out1[s]) << std::endl; - } - - - delete [] spinorIn; - delete [] spinorOut; - } + const multi1d& latdims = Layout::subgridLattSize(); + int halfsize=latdims[0]*latdims[1]*latdims[2]*latdims[3]/2; + int fermsize=halfsize*Nc*Ns*2; + + // In1 is input to Chroma, Out1 is result + multi1d in1( this->size() ); + multi1d out1(this->size() ); + + // In2 is input to QUDA, Out2 is result + multi1d in2( this->size() ); + multi1d out2(this->size() ); + + + for(int s=0; s < this->size(); s++ ) { + gaussian(in1[s]); // Gaussian into in1 + in2[s] = in1[s]; // copy to in2 + + } + + for(int d=0; d < 2; d++) { + for(int s=0; s < this->size(); s++ ) { + out1[s]=zero; // zero both out1 and out2 + out2[s]=zero; + } + + if ( d==0 ) { + // Apply A to in2 + QDPIO::cout << "DOing Mat" << std::endl; + (*A)(out2, in2, PLUS); + } + else { + QDPIO::cout << "Doing MatDag" << std::endl; + (*A)(out2, in2, MINUS); + } + + // Copy in1 into QUDA + REAL* spinorIn = new REAL[quda_inv_param.Ls*fermsize]; + REAL* spinorOut = new REAL[quda_inv_param.Ls*fermsize]; + memset((spinorIn), 0, fermsize*quda_inv_param.Ls*sizeof(REAL)); + memset((spinorOut), 0, fermsize*quda_inv_param.Ls*sizeof(REAL)); + + for(unsigned int s=0; s(&(psi_s[s].elem(all.start()).elem(0).elem(0).real())),0,fermsize*2*sizeof(REAL)); + memcpy((&out1[s].elem(rb[1].start()).elem(0).elem(0).real()),(&spinorOut[fermsize*s]),fermsize*sizeof(REAL)); + } + + // Now compare out1 and out2 + for(int s=0; s < this->size();s++) { + out1[s] *= invTwoKappaB; + + QDPIO::cout << "s=" << s << " diff=" << norm2(out2[s]-out1[s]) << std::endl; + } + + + delete [] spinorIn; + delete [] spinorOut; + } } @@ -496,33 +501,33 @@ namespace Chroma QDPIO::cout << "TwoKappa = " << twoKappaB << " invTwoKappa_b = " << invTwoKappaB << std::endl; if ( invParam.axialGaugeP ) { - multi1d g_chi( this->size()); - multi1d g_psi( this->size()); - - // Gauge Fix source and initial guess - QDPIO::cout << "Gauge Fixing source and initial guess" << std::endl; - for(int s=0; ssize(); s++) { - g_psi[s][rb[1]] *= twoKappaB; - } - - QDPIO::cout << "Untransforming solution." << std::endl; - for(int s=0; s g_chi( this->size()); + multi1d g_psi( this->size()); + + // Gauge Fix source and initial guess + QDPIO::cout << "Gauge Fixing source and initial guess" << std::endl; + for(int s=0; ssize(); s++) { + g_psi[s][rb[1]] *= twoKappaB; + } + + QDPIO::cout << "Untransforming solution." << std::endl; + for(int s=0; ssize(); s++) { - psi[s][rb[1]] *= twoKappaB; - } + QDPIO::cout << "Calling qudaInvert" << std::endl; + res = qudaInvert(chi,psi); + for(int s=0; s < this->size(); s++) { + psi[s][rb[1]] *= twoKappaB; + } } @@ -532,34 +537,34 @@ namespace Chroma // Check Solution { - multi1d r(A->size()); - multi1d Ax(A->size()); - r=zero; - Ax=zero; - Double r_norm(zero); - Double b_norm(zero); - (*A)(Ax, psi, PLUS); - for(int s=0; s < A->size(); s++) { - r[s][rb[1]] = chi[s] - Ax[s]; - r_norm += norm2(r[s], rb[1]); - b_norm += norm2(chi[s], rb[1]); - } - - Double resid = sqrt(r_norm); - Double rel_resid = sqrt(r_norm/b_norm); - - res.resid = resid; - QDPIO::cout << "QUDA_"<< solver_string <<"_NEF_SOLVER: " << res.n_count << " iterations. Max. Rsd = " << res.resid << " Max. Relative Rsd = " << rel_resid << std::endl; - - - // Convergence Check/Blow Up - if ( ! invParam.SilentFailP ) { - if ( toBool( rel_resid > invParam.RsdToleranceFactor*invParam.RsdTarget) ) { - QDPIO::cerr << "ERROR: QUDA Solver residuum is outside tolerance: QUDA resid="<< rel_resid << " Desired =" << invParam.RsdTarget << " Max Tolerated = " << invParam.RsdToleranceFactor*invParam.RsdTarget << std::endl; - QDP_abort(1); - } - - } + multi1d r(A->size()); + multi1d Ax(A->size()); + r=zero; + Ax=zero; + Double r_norm(zero); + Double b_norm(zero); + (*A)(Ax, psi, PLUS); + for(int s=0; s < A->size(); s++) { + r[s][rb[1]] = chi[s] - Ax[s]; + r_norm += norm2(r[s], rb[1]); + b_norm += norm2(chi[s], rb[1]); + } + + Double resid = sqrt(r_norm); + Double rel_resid = sqrt(r_norm/b_norm); + + res.resid = resid; + QDPIO::cout << "QUDA_"<< solver_string <<"_NEF_SOLVER: " << res.n_count << " iterations. Max. Rsd = " << res.resid << " Max. Relative Rsd = " << rel_resid << std::endl; + + + // Convergence Check/Blow Up + if ( ! invParam.SilentFailP ) { + if ( toBool( rel_resid > invParam.RsdToleranceFactor*invParam.RsdTarget) ) { + QDPIO::cerr << "ERROR: QUDA Solver residuum is outside tolerance: QUDA resid="<< rel_resid << " Desired =" << invParam.RsdTarget << " Max Tolerated = " << invParam.RsdToleranceFactor*invParam.RsdTarget << std::endl; + QDP_abort(1); + } + + } } @@ -572,23 +577,23 @@ namespace Chroma LinOpSysSolverQUDANEF() {} #if 1 - Q links_orig; + Q links_orig; #endif - U GFixMat; - QudaPrecision_s cpu_prec; - QudaPrecision_s gpu_prec; - QudaPrecision_s gpu_half_prec; + U GFixMat; + QudaPrecision_s cpu_prec; + QudaPrecision_s gpu_prec; + QudaPrecision_s gpu_half_prec; - Handle< LinearOperatorArray > A; - const SysSolverQUDANEFParams invParam; - QudaGaugeParam q_gauge_param; - mutable QudaInvertParam quda_inv_param; + Handle< LinearOperatorArray > A; + const SysSolverQUDANEFParams invParam; + QudaGaugeParam q_gauge_param; + mutable QudaInvertParam quda_inv_param; - SystemSolverResults_t qudaInvert(const multi1d& chi_s, multi1d& psi_s)const; + SystemSolverResults_t qudaInvert(const multi1d& chi_s, multi1d& psi_s)const; - std::string solver_string; - }; + std::string solver_string; + }; } // End namespace diff --git a/lib/actions/ferm/invert/quda_solvers/syssolver_quda_nef_params.cc b/lib/actions/ferm/invert/quda_solvers/syssolver_quda_nef_params.cc index a1fd86338..4c79e8c4e 100644 --- a/lib/actions/ferm/invert/quda_solvers/syssolver_quda_nef_params.cc +++ b/lib/actions/ferm/invert/quda_solvers/syssolver_quda_nef_params.cc @@ -10,7 +10,7 @@ using namespace QDP; namespace Chroma { SysSolverQUDANEFParams::SysSolverQUDANEFParams(XMLReader& xml, - const std::string& path) + const std::string& path) { XMLReader paramtop(xml, path); @@ -30,6 +30,12 @@ namespace Chroma { else { verboseP = false; } + if ( paramtop.count("Debug") > 0 ) { // add debug bool to support QUDA_DEBUG_VERBOSE + read(paramtop, "Debug", debugP); + } + else { + debugP = false; + } if ( paramtop.count("AsymmetricLinop") > 0 ) { read(paramtop, "AsymmetricLinop", asymmetricP); } @@ -80,10 +86,10 @@ namespace Chroma { } if( paramtop.count("RsdToleranceFactor") > 0 ) { - read(paramtop, "RsdToleranceFactor", RsdToleranceFactor); + read(paramtop, "RsdToleranceFactor", RsdToleranceFactor); } else { - RsdToleranceFactor = Real(10); // Tolerate an order of magnitude difference by default. + RsdToleranceFactor = Real(10); // Tolerate an order of magnitude difference by default. } if( paramtop.count("AutotuneDslash") > 0 ) { @@ -122,29 +128,29 @@ namespace Chroma { } if ( paramtop.count("DoCGNR") > 0 ) { - read(paramtop, "DoCGNR", cgnrP); + read(paramtop, "DoCGNR", cgnrP); } else { - cgnrP = false ; // Do CGNE by default + cgnrP = false ; // Do CGNE by default } if( paramtop.count("Pipeline") > 0 ) { - read(paramtop, "Pipeline", Pipeline); + read(paramtop, "Pipeline", Pipeline); } else { - Pipeline = 1; + Pipeline = 0; // For CG solvers, this should be 0, not 1 } } void read(XMLReader& xml, const std::string& path, - SysSolverQUDANEFParams& p) + SysSolverQUDANEFParams& p) { SysSolverQUDANEFParams tmp(xml, path); p = tmp; } void write(XMLWriter& xml, const std::string& path, - const SysSolverQUDANEFParams& p) { + const SysSolverQUDANEFParams& p) { push(xml, path); write(xml, "MaxIter", p.MaxIter); write(xml, "RsdTarget", p.RsdTarget); @@ -153,6 +159,7 @@ namespace Chroma { write(xml, "Delta", p.Delta); write(xml, "SolverType", p.solverType); write(xml, "Verbose", p.verboseP); + write(xml, "Debug", p.debugP); write(xml, "AsymmetricLinop", p.asymmetricP); write(xml, "CudaPrecision", p.cudaPrecision); write(xml, "CudaReconstruct", p.cudaReconstruct); diff --git a/lib/actions/ferm/invert/quda_solvers/syssolver_quda_nef_params.h b/lib/actions/ferm/invert/quda_solvers/syssolver_quda_nef_params.h index 004bb4dd9..02e1b3e65 100644 --- a/lib/actions/ferm/invert/quda_solvers/syssolver_quda_nef_params.h +++ b/lib/actions/ferm/invert/quda_solvers/syssolver_quda_nef_params.h @@ -26,6 +26,7 @@ namespace Chroma RsdToleranceFactor = Real(10); //< Tolerate if the solution achived is better (less) than rsdToleranceFactor*RsdTarget tuneDslashP = false ; //< v0.3 autotune feature verboseP = false; + debugP = false; // if true and verboseP true, use QUDA_DEBUG_VERBOSE innerParamsP = false; backup_invP = false; dump_on_failP = false; @@ -41,6 +42,7 @@ namespace Chroma Delta = p.Delta; solverType = p.solverType; verboseP = p.verboseP; + debugP = p.debugP; asymmetricP = p.asymmetricP; cudaPrecision = p.cudaPrecision; cudaReconstruct = p.cudaReconstruct; @@ -67,6 +69,7 @@ namespace Chroma Real Delta; QudaSolverType solverType; bool verboseP; + bool debugP; bool asymmetricP; QudaPrecisionType cudaPrecision; QudaReconsType cudaReconstruct; @@ -77,7 +80,7 @@ namespace Chroma Real RsdToleranceFactor; bool tuneDslashP; bool innerParamsP; - + // GCR Specific params // Params for the preconditioner Handle innerParams; @@ -94,7 +97,7 @@ namespace Chroma void read(XMLReader& xml, const std::string& path, SysSolverQUDANEFParams& p); void write(XMLWriter& xml, const std::string& path, - const SysSolverQUDANEFParams& param); + const SysSolverQUDANEFParams& param);