diff options
Diffstat (limited to 'src/lib')
-rw-r--r-- | src/lib/netlist/build/makefile | 4 | ||||
-rw-r--r-- | src/lib/netlist/devices/nld_system.h | 6 | ||||
-rw-r--r-- | src/lib/netlist/nl_base.cpp | 52 | ||||
-rw-r--r-- | src/lib/netlist/nl_base.h | 36 | ||||
-rw-r--r-- | src/lib/netlist/nl_setup.cpp | 5 | ||||
-rw-r--r-- | src/lib/netlist/nl_util.h | 41 | ||||
-rw-r--r-- | src/lib/netlist/plib/pstate.h | 45 | ||||
-rw-r--r-- | src/lib/netlist/prg/nltool.cpp | 9 | ||||
-rw-r--r-- | src/lib/netlist/solver/nld_matrix_solver.h | 106 | ||||
-rw-r--r-- | src/lib/netlist/solver/nld_ms_direct.h | 212 | ||||
-rw-r--r-- | src/lib/netlist/solver/nld_ms_direct1.h | 4 | ||||
-rw-r--r-- | src/lib/netlist/solver/nld_ms_direct2.h | 4 | ||||
-rw-r--r-- | src/lib/netlist/solver/nld_ms_direct_lu.h | 11 | ||||
-rw-r--r-- | src/lib/netlist/solver/nld_ms_sm.h | 192 | ||||
-rw-r--r-- | src/lib/netlist/solver/nld_ms_sor_mat.h | 4 | ||||
-rw-r--r-- | src/lib/netlist/solver/nld_ms_w.h | 190 | ||||
-rw-r--r-- | src/lib/netlist/solver/nld_solver.cpp | 77 | ||||
-rw-r--r-- | src/lib/util/chdcodec.cpp | 29 | ||||
-rw-r--r-- | src/lib/util/un7z.cpp | 15 | ||||
-rw-r--r-- | src/lib/util/unzip.cpp | 858 | ||||
-rw-r--r-- | src/lib/util/zippath.cpp | 98 |
21 files changed, 996 insertions, 1002 deletions
diff --git a/src/lib/netlist/build/makefile b/src/lib/netlist/build/makefile index be49d098a0d..684b2687ba1 100644 --- a/src/lib/netlist/build/makefile +++ b/src/lib/netlist/build/makefile @@ -20,8 +20,8 @@ CDEFS = -DPSTANDALONE=1 -DPTR64=1 #-Werror #CFLAGS = $(LTO) -g -O3 -std=c++98 -Doverride="" -march=native -msse4.2 -Wall -Wpedantic -Wsign-compare -Wextra -Wno-long-long -Wno-unused-parameter -Wno-unused-result -Wno-variadic-macros -I.. #LDFLAGS = $(LTO) -g -O3 -std=c++98 -CFLAGS = $(LTO) -g -O3 -std=c++11 -I.. -Wall -Wpedantic -Wsign-compare -Wextra -Wno-unused-parameter -LDFLAGS = $(LTO) -g -O3 -std=c++11 +CFLAGS = $(LTO) -g -O3 -std=c++11 -march=native -I.. -Wall -Wpedantic -Wsign-compare -Wextra -Wno-unused-parameter +LDFLAGS = $(LTO) -g -O3 -std=c++11 -lpthread CC = @g++-5 LD = @g++-5 diff --git a/src/lib/netlist/devices/nld_system.h b/src/lib/netlist/devices/nld_system.h index 8ea3640ec14..7b9cb7298e5 100644 --- a/src/lib/netlist/devices/nld_system.h +++ b/src/lib/netlist/devices/nld_system.h @@ -479,10 +479,10 @@ public: const pstring &def_param) : base_factory_t(name, classname, def_param), m_setup(setup) { } - class dummy : public device_t + class wrapper : public device_t { public: - dummy(const pstring &dev_name) : device_t(), m_dev_name(dev_name) { } + wrapper(const pstring &dev_name) : device_t(), m_dev_name(dev_name) { } protected: virtual void init(netlist_t &anetlist, const pstring &aname) override { @@ -499,7 +499,7 @@ public: ATTR_COLD device_t *Create() override { - device_t *r = palloc(dummy(this->name())); + device_t *r = palloc(wrapper(this->name())); return r; } diff --git a/src/lib/netlist/nl_base.cpp b/src/lib/netlist/nl_base.cpp index 7afa0285ef8..eb83749c79b 100644 --- a/src/lib/netlist/nl_base.cpp +++ b/src/lib/netlist/nl_base.cpp @@ -74,13 +74,12 @@ logic_family_desc_t *family_CD4XXX = palloc(logic_family_cd4xxx_t); queue_t::queue_t(netlist_t &nl) : timed_queue<net_t *, netlist_time>(512) - , object_t(QUEUE, GENERIC) + , object_t(nl, "QUEUE", QUEUE, GENERIC) , pstate_callback_t() , m_qsize(0) , m_times(512) , m_names(512) { - this->init_object(nl, "QUEUE"); } void queue_t::register_state(pstate_manager_t &manager, const pstring &module) @@ -128,13 +127,33 @@ void queue_t::on_post_load() ATTR_COLD object_t::object_t(const type_t atype, const family_t afamily) : m_objtype(atype) , m_family(afamily) -, m_netlist(NULL) +, m_netlist(nullptr) {} +ATTR_COLD object_t::object_t(netlist_t &nl, const type_t atype, const family_t afamily) +: m_objtype(atype) +, m_family(afamily) +, m_netlist(&nl) +{} + +ATTR_COLD object_t::object_t(netlist_t &nl, const pstring &aname, const type_t atype, const family_t afamily) +: m_name(aname) +, m_objtype(atype) +, m_family(afamily) +, m_netlist(&nl) +{ +} + ATTR_COLD object_t::~object_t() { } +ATTR_COLD void object_t::init_object(const pstring &aname) +{ + m_name = aname; + save_register(); +} + ATTR_COLD void object_t::init_object(netlist_t &nl, const pstring &aname) { m_netlist = &nl; @@ -171,8 +190,8 @@ ATTR_COLD void device_object_t::init_object(core_device_t &dev, // netlist_t // ---------------------------------------------------------------------------------------- -netlist_t::netlist_t() - : object_t(NETLIST, GENERIC), pstate_manager_t(), +netlist_t::netlist_t(const pstring &aname) + : pstate_manager_t(), m_stop(netlist_time::zero), m_time(netlist_time::zero), m_use_deactivate(0), @@ -181,9 +200,12 @@ netlist_t::netlist_t() m_solver(NULL), m_gnd(NULL), m_params(NULL), + m_name(aname), m_setup(NULL), m_log(this) { + save_item(static_cast<pstate_callback_t &>(m_queue), this, "m_queue"); + save_item(m_time, this, "m_time"); } netlist_t::~netlist_t() @@ -202,13 +224,6 @@ netlist_t::~netlist_t() pstring::resetmem(); } -ATTR_COLD void netlist_t::save_register() -{ - save(static_cast<pstate_callback_t &>(m_queue), "m_queue"); - save(NLNAME(m_time)); - object_t::save_register(); -} - ATTR_HOT nl_double netlist_t::gmin() const { return solver()->gmin(); @@ -925,6 +940,19 @@ ATTR_COLD analog_output_t::analog_output_t() net().m_cur_Analog = NL_FCONST(0.99); } +ATTR_COLD analog_output_t::analog_output_t(core_device_t &dev, const pstring &aname) + : analog_t(OUTPUT), m_proxied_net(NULL) +{ + this->set_net(m_my_net); + set_state(STATE_OUT); + + net().m_cur_Analog = NL_FCONST(0.99); + + analog_t::init_object(dev, aname); + net().init_object(dev.netlist(), aname + ".net"); + net().register_railterminal(*this); +} + ATTR_COLD void analog_output_t::init_object(core_device_t &dev, const pstring &aname) { analog_t::init_object(dev, aname); diff --git a/src/lib/netlist/nl_base.h b/src/lib/netlist/nl_base.h index 52a91704607..3b6eb7b75e1 100644 --- a/src/lib/netlist/nl_base.h +++ b/src/lib/netlist/nl_base.h @@ -367,7 +367,7 @@ namespace netlist // object_t // ----------------------------------------------------------------------------- - class object_t + class object_t : public pstate_interface_t<object_t> { P_PREVENT_COPYING(object_t) public: @@ -378,8 +378,7 @@ namespace netlist PARAM = 3, NET = 4, DEVICE = 5, - NETLIST = 6, - QUEUE = 7 + QUEUE = 6 }; enum family_t { // Terminal families @@ -405,15 +404,18 @@ namespace netlist }; ATTR_COLD object_t(const type_t atype, const family_t afamily); + ATTR_COLD object_t(netlist_t &nl, const type_t atype, const family_t afamily); + ATTR_COLD object_t(netlist_t &nl, const pstring &aname, const type_t atype, const family_t afamily); virtual ~object_t(); + ATTR_COLD void init_object(const pstring &aname); ATTR_COLD void init_object(netlist_t &nl, const pstring &aname); ATTR_COLD bool isInitialized() { return (m_netlist != NULL); } ATTR_COLD const pstring &name() const; - PSTATE_INTERFACE_DECL() + ATTR_COLD inline pstate_manager_t *state_manager(); ATTR_HOT type_t type() const { return m_objtype; } ATTR_HOT family_t family() const { return m_family; } @@ -904,6 +906,7 @@ namespace netlist P_PREVENT_COPYING(analog_output_t) public: + ATTR_COLD analog_output_t(core_device_t &dev, const pstring &aname); ATTR_COLD analog_output_t(); ATTR_COLD void init_object(core_device_t &dev, const pstring &aname); @@ -1183,14 +1186,16 @@ namespace netlist // ----------------------------------------------------------------------------- - class netlist_t : public object_t, public pstate_manager_t, public plog_dispatch_intf + class netlist_t : public pstate_manager_t, public plog_dispatch_intf { P_PREVENT_COPYING(netlist_t) public: - netlist_t(); + netlist_t(const pstring &aname); virtual ~netlist_t(); + pstring name() const { return m_name; } + ATTR_COLD void start(); ATTR_COLD void stop(); @@ -1265,14 +1270,12 @@ namespace netlist pnamedlist_t<core_device_t *> m_started_devices; #endif - ATTR_COLD plog_base<NL_DEBUG> &log() { return m_log; } - ATTR_COLD const plog_base<NL_DEBUG> &log() const { return m_log; } + ATTR_COLD plog_base<NL_DEBUG> &log() { return m_log; } + ATTR_COLD const plog_base<NL_DEBUG> &log() const { return m_log; } - protected: + virtual void reset(); - /* from object */ - virtual void reset() override; - virtual void save_register() override; + protected: #if (NL_KEEP_STATISTICS) // performance @@ -1288,13 +1291,13 @@ namespace netlist bool m_use_deactivate; queue_t m_queue; - devices::NETLIB_NAME(mainclock) * m_mainclock; devices::NETLIB_NAME(solver) * m_solver; devices::NETLIB_NAME(gnd) * m_gnd; - devices::NETLIB_NAME(netlistparams) *m_params; + + pstring m_name; setup_t *m_setup; plog_base<NL_DEBUG> m_log; }; @@ -1303,7 +1306,10 @@ namespace netlist // inline implementations // ----------------------------------------------------------------------------- - PSTATE_INTERFACE(object_t, m_netlist, name()) + ATTR_COLD inline pstate_manager_t *object_t::state_manager() + { + return m_netlist; + } template <class C, param_t::param_type_t T> ATTR_HOT inline void param_template_t<C, T>::setTo(const C ¶m) diff --git a/src/lib/netlist/nl_setup.cpp b/src/lib/netlist/nl_setup.cpp index 399fe86d213..dad17991b87 100644 --- a/src/lib/netlist/nl_setup.cpp +++ b/src/lib/netlist/nl_setup.cpp @@ -197,8 +197,6 @@ pstring setup_t::objtype_as_astr(object_t &in) const return "PARAM"; case terminal_t::DEVICE: return "DEVICE"; - case terminal_t::NETLIST: - return "NETLIST"; case terminal_t::QUEUE: return "QUEUE"; } @@ -283,9 +281,6 @@ void setup_t::register_object(device_t &dev, const pstring &name, object_t &obj) case terminal_t::DEVICE: log().fatal("Device registration not yet supported - {1}\n", name); break; - case terminal_t::NETLIST: - log().fatal("Netlist registration not yet supported - {1}\n", name); - break; case terminal_t::QUEUE: log().fatal("QUEUE registration not yet supported - {1}\n", name); break; diff --git a/src/lib/netlist/nl_util.h b/src/lib/netlist/nl_util.h index 87ec2270231..50c545c22e3 100644 --- a/src/lib/netlist/nl_util.h +++ b/src/lib/netlist/nl_util.h @@ -11,6 +11,7 @@ #include <cmath> #include <cstring> #include <cstdlib> +#include <algorithm> #include "plib/pstring.h" #include "plib/plists.h" @@ -40,26 +41,35 @@ private: nl_math() {}; public: - ATTR_HOT inline static float exp(const float x) { return std::exp(x); } - ATTR_HOT inline static double abs(const double x) { return std::fabs(x); } - ATTR_HOT inline static float abs(const float x) { return std::fabs(x); } - ATTR_HOT inline static double log(const double x) { return std::log(x); } - ATTR_HOT inline static float log(const float x) { return std::log(x); } -#if defined(_MSC_VER) && _MSC_VER < 1800 - ATTR_HOT inline static double e_log1p(const double x) { return nl_math::log(1.0 + x); } - ATTR_HOT inline static float e_log1p(const float x) { return nl_math::log(1.0 + x); } + template <typename T> + static T abs(const T &x) { return std::abs(x); } + + template <typename T> + static T max(const T &x, const T &y) { return std::max(x, y); } + + template <typename T> + static T min(const T &x, const T &y) { return std::min(x, y); } + + template <typename T> + static T log(const T &x) { return std::log(x); } + + #if defined(_MSC_VER) && _MSC_VER < 1800 + ATTR_HOT inline static double e_log1p(const double &x) { return nl_math::log(1.0 + x); } + ATTR_HOT inline static float e_log1p(const float &x) { return nl_math::log(1.0 + x); } #else - ATTR_HOT inline static double e_log1p(const double x) { return log1p(x); } - ATTR_HOT inline static float e_log1p(const float x) { return log1pf(x); } + template <typename T> + static T e_log1p(const T &x) { return log1p(x); } #endif - ATTR_HOT inline static double sqrt(const double x) { return std::sqrt(x); } - ATTR_HOT inline static float sqrt(const float x) { return std::sqrt(x); } + + template <typename T> + static T sqrt(const T &x) { return std::sqrt(x); } // this one has an accuracy of better than 5%. That's enough for our purpose // add c3 and it'll be better than 1% #if 0 - inline static double fastexp_h(const double x) + ATTR_HOT inline static float exp(const float &x) { return std::exp(x); } + inline static double fastexp_h(const double &x) { /* static */ const double ln2r = 1.442695040888963387; /* static */ const double ln2 = 0.693147180559945286; @@ -79,7 +89,7 @@ public: return pow(2.0, t)*e; } - ATTR_HOT inline static double exp(const double x) + ATTR_HOT inline static double exp(const double &x) { if (x<0) return 1.0 / fastexp_h(-x); @@ -87,7 +97,8 @@ public: return fastexp_h(x); } #else - ATTR_HOT inline static double exp(const double x) { return std::exp(x); } + template <typename T> + static double exp(const T &x) { return std::exp(x); } #endif }; diff --git a/src/lib/netlist/plib/pstate.h b/src/lib/netlist/plib/pstate.h index 91c67ee0e1c..88c0c005927 100644 --- a/src/lib/netlist/plib/pstate.h +++ b/src/lib/netlist/plib/pstate.h @@ -15,19 +15,6 @@ // state saving ... // ---------------------------------------------------------------------------------------- -#define PSTATE_INTERFACE_DECL() \ - template<typename C> ATTR_COLD void save(C &state, const pstring &stname); \ - template<typename C, std::size_t N> ATTR_COLD void save(C (&state)[N], const pstring &stname); \ - template<typename C> ATTR_COLD void save(C *state, const pstring &stname, const int count); - -#define PSTATE_INTERFACE(obj, manager, module) \ - template<typename C> ATTR_COLD void obj::save(C &state, const pstring &stname) \ - { manager->save_item(state, this, module + "." + stname); } \ - template<typename C, std::size_t N> ATTR_COLD void obj::save(C (&state)[N], const pstring &stname) \ - { manager->save_state_ptr(module + "." + stname, pstate_datatype<C>::type, this, sizeof(state[0]), N, &(state[0]), false); } \ - template<typename C> ATTR_COLD void obj::save(C *state, const pstring &stname, const int count) \ - { manager->save_state_ptr(module + "." + stname, pstate_datatype<C>::type, this, sizeof(C), count, state, false); } - enum pstate_data_type_e { NOT_SUPPORTED, DT_CUSTOM, @@ -148,6 +135,12 @@ public: save_state_ptr(stname, pstate_datatype<C>::type, owner, sizeof(C), count, state, false); } + template<typename C> + void save_item(std::vector<C> &v, const void *owner, const pstring &stname) + { + save_state(v.data(), owner, stname, v.size()); + } + ATTR_COLD void pre_save(); ATTR_COLD void post_load(); ATTR_COLD void remove_save_items(const void *owner); @@ -164,5 +157,31 @@ private: template<> ATTR_COLD void pstate_manager_t::save_item(pstate_callback_t &state, const void *owner, const pstring &stname); +template <typename T> +class pstate_interface_t +{ +public: + pstate_interface_t() { } + + template<typename C> void save(C &state, const pstring &stname) + { + pstate_manager_t *manager = static_cast<T*>(this)->state_manager(); + pstring module = static_cast<T*>(this)->name(); + manager->save_item(state, this, module + "." + stname); + } + template<typename C, std::size_t N> ATTR_COLD void save(C (&state)[N], const pstring &stname) + { + pstate_manager_t *manager = static_cast<T*>(this)->state_manager(); + pstring module = static_cast<T*>(this)->name(); + manager->save_state_ptr(module + "." + stname, pstate_datatype<C>::type, this, sizeof(state[0]), N, &(state[0]), false); + } + template<typename C> ATTR_COLD void save(C *state, const pstring &stname, const int count) + { + pstate_manager_t *manager = static_cast<T*>(this)->state_manager(); + pstring module = static_cast<T*>(this)->name(); + manager->save_state_ptr(module + "." + stname, pstate_datatype<C>::type, this, sizeof(C), count, state, false); + } +}; + #endif /* PSTATE_H_ */ diff --git a/src/lib/netlist/prg/nltool.cpp b/src/lib/netlist/prg/nltool.cpp index ea5211a3c87..6d34683731e 100644 --- a/src/lib/netlist/prg/nltool.cpp +++ b/src/lib/netlist/prg/nltool.cpp @@ -145,8 +145,8 @@ class netlist_tool_t : public netlist::netlist_t { public: - netlist_tool_t() - : netlist::netlist_t(), m_opts(NULL), m_setup(NULL) + netlist_tool_t(const pstring &aname) + : netlist::netlist_t(aname), m_opts(NULL), m_setup(NULL) { } @@ -159,7 +159,6 @@ public: void init() { m_setup = palloc(netlist::setup_t(this)); - this->init_object(*this, "netlist"); m_setup->init(); } @@ -281,7 +280,7 @@ pvector_t<input_t> *read_input(netlist::netlist_t *netlist, pstring fname) static void run(tool_options_t &opts) { - netlist_tool_t nt; + netlist_tool_t nt("netlist"); osd_ticks_t t = osd_ticks(); nt.m_opts = &opts; @@ -326,7 +325,7 @@ static void run(tool_options_t &opts) static void listdevices() { - netlist_tool_t nt; + netlist_tool_t nt("netlist"); nt.init(); const netlist::factory_list_t &list = nt.setup().factory(); diff --git a/src/lib/netlist/solver/nld_matrix_solver.h b/src/lib/netlist/solver/nld_matrix_solver.h index 715eab6756a..58947144ff5 100644 --- a/src/lib/netlist/solver/nld_matrix_solver.h +++ b/src/lib/netlist/solver/nld_matrix_solver.h @@ -10,14 +10,16 @@ #include "solver/nld_solver.h" +#include <type_traits> + NETLIB_NAMESPACE_DEVICES_START() class terms_t { P_PREVENT_COPYING(terms_t) - public: - ATTR_COLD terms_t() : m_railstart(0) +public: + ATTR_COLD terms_t() : m_railstart(0), m_last_V(0) {} ATTR_COLD void clear() @@ -48,6 +50,10 @@ class terms_t pvector_t<unsigned> m_nz; /* all non zero for multiplication */ pvector_t<unsigned> m_nzrd; /* non zero right of the diagonal for elimination, may include RHS element */ pvector_t<unsigned> m_nzbd; /* non zero below of the diagonal for elimination */ + + /* state */ + nl_double m_last_V; + private: pvector_t<terminal_t *> m_term; pvector_t<int> m_net_other; @@ -55,6 +61,7 @@ private: pvector_t<nl_double> m_gt; pvector_t<nl_double> m_Idr; pvector_t<nl_double *> m_other_curanalog; + }; class matrix_solver_t : public device_t @@ -104,14 +111,28 @@ protected: ATTR_COLD void setup_base(analog_net_t::list_t &nets); void update_dynamic(); - virtual void add_term(int net_idx, terminal_t *term) = 0; virtual void vsetup(analog_net_t::list_t &nets) = 0; virtual int vsolve_non_dynamic(const bool newton_raphson) = 0; - virtual netlist_time compute_next_timestep() = 0; + /* virtual */ netlist_time compute_next_timestep(); + /* virtual */ void add_term(int net_idx, terminal_t *term); + + template <typename T> + void store(const T * RESTRICT V); + template <typename T> + T delta(const T * RESTRICT V); + + template <typename T> + void build_LE_A(T & child); + template <typename T> + void build_LE_RHS(T & child); + + pvector_t<terms_t *> m_terms; pvector_t<analog_net_t *> m_nets; pvector_t<analog_output_t *> m_inps; + pvector_t<terms_t *> m_rails_temp; + int m_stat_calculations; int m_stat_newton_raphson; int m_stat_vsolver_calls; @@ -138,6 +159,83 @@ private: const eSolverType m_type; }; +template <typename T> +T matrix_solver_t::delta(const T * RESTRICT V) +{ + /* FIXME: Ideally we should also include currents (RHS) here. This would + * need a revaluation of the right hand side after voltages have been updated + * and thus belong into a different calculation. This applies to all solvers. + */ + + const unsigned iN = this->m_terms.size(); + T cerr = 0; + for (unsigned i = 0; i < iN; i++) + cerr = nl_math::max(cerr, nl_math::abs(V[i] - (T) this->m_nets[i]->m_cur_Analog)); + return cerr; +} + +template <typename T> +void matrix_solver_t::store(const T * RESTRICT V) +{ + for (unsigned i = 0, iN=m_terms.size(); i < iN; i++) + this->m_nets[i]->m_cur_Analog = V[i]; +} + +template <typename T> +void matrix_solver_t::build_LE_A(T & child) +{ + static_assert(std::is_base_of<matrix_solver_t, T>::value, "T must derive from matrix_solver_t"); + + const unsigned iN = child.N(); + for (unsigned k = 0; k < iN; k++) + { + for (unsigned i=0; i < iN; i++) + child.A(k,i) = 0.0; + + const unsigned terms_count = m_terms[k]->count(); + const unsigned railstart = m_terms[k]->m_railstart; + const nl_double * RESTRICT gt = m_terms[k]->gt(); + + { + nl_double akk = 0.0; + for (unsigned i = 0; i < terms_count; i++) + akk += gt[i]; + + child.A(k,k) = akk; + } + + const nl_double * RESTRICT go = m_terms[k]->go(); + const int * RESTRICT net_other = m_terms[k]->net_other(); + + for (unsigned i = 0; i < railstart; i++) + child.A(k,net_other[i]) -= go[i]; + } +} + +template <typename T> +void matrix_solver_t::build_LE_RHS(T & child) +{ + const unsigned iN = child.N(); + for (unsigned k = 0; k < iN; k++) + { + nl_double rhsk_a = 0.0; + nl_double rhsk_b = 0.0; + + const unsigned terms_count = m_terms[k]->count(); + const nl_double * RESTRICT go = m_terms[k]->go(); + const nl_double * RESTRICT Idr = m_terms[k]->Idr(); + const nl_double * const * RESTRICT other_cur_analog = m_terms[k]->other_curanalog(); + + for (unsigned i = 0; i < terms_count; i++) + rhsk_a = rhsk_a + Idr[i]; + + for (unsigned i = m_terms[k]->m_railstart; i < terms_count; i++) + //rhsk = rhsk + go[i] * terms[i]->m_otherterm->net().as_analog().Q_Analog(); + rhsk_b = rhsk_b + go[i] * *other_cur_analog[i]; + + child.RHS(k) = rhsk_a + rhsk_b; + } +} NETLIB_NAMESPACE_DEVICES_END() diff --git a/src/lib/netlist/solver/nld_ms_direct.h b/src/lib/netlist/solver/nld_ms_direct.h index f0c892ab698..2b0ae80b855 100644 --- a/src/lib/netlist/solver/nld_ms_direct.h +++ b/src/lib/netlist/solver/nld_ms_direct.h @@ -50,8 +50,7 @@ struct ti_t }; static ti_t ti[MAXTHR]; -//static std::thread thr[MAXTHR]; -static std::thread thr[num_thr]; +static std::thread thr[MAXTHR]; int thr_init = 0; @@ -119,6 +118,7 @@ class matrix_solver_direct_t: public matrix_solver_t, public thr_intf class matrix_solver_direct_t: public matrix_solver_t #endif { + friend class matrix_solver_t; public: matrix_solver_direct_t(const solver_parameters_t *params, const int size); @@ -130,27 +130,16 @@ public: virtual void reset() override { matrix_solver_t::reset(); } protected: - virtual void add_term(int net_idx, terminal_t *term) override; virtual int vsolve_non_dynamic(const bool newton_raphson) override; int solve_non_dynamic(const bool newton_raphson); inline unsigned N() const { if (m_N == 0) return m_dim; else return m_N; } - void build_LE_A(); - void build_LE_RHS(); void LE_solve(); template <typename T> void LE_back_subst(T * RESTRICT x); - template <typename T> - T delta(const T * RESTRICT V); - - template <typename T> - void store(const T * RESTRICT V); - - virtual netlist_time compute_next_timestep() override; - #if TEST_PARALLEL int x_i[10]; int x_start[10]; @@ -170,10 +159,6 @@ protected: inline nl_ext_double &RHS(const T1 &r) { return m_A[r][N()]; } #endif ATTR_ALIGN nl_double m_last_RHS[_storage_N]; // right hand side - contains currents - ATTR_ALIGN nl_double m_last_V[_storage_N]; - - terms_t * m_terms[_storage_N]; - terms_t *m_rails_temp; private: static const std::size_t m_pitch = (((_storage_N + 1) + 7) / 8) * 8; @@ -197,109 +182,36 @@ private: template <unsigned m_N, unsigned _storage_N> matrix_solver_direct_t<m_N, _storage_N>::~matrix_solver_direct_t() { - for (unsigned k = 0; k < N(); k++) - { - pfree(m_terms[k]); - } - pfree_array(m_rails_temp); #if (NL_USE_DYNAMIC_ALLOCATION) pfree_array(m_A); #endif #if TEST_PARALLEL thr_dispose(); #endif - } template <unsigned m_N, unsigned _storage_N> -netlist_time matrix_solver_direct_t<m_N, _storage_N>::compute_next_timestep() -{ - nl_double new_solver_timestep = m_params.m_max_timestep; - - if (m_params.m_dynamic) - { - /* - * FIXME: We should extend the logic to use either all nets or - * only output nets. - */ - for (unsigned k = 0, iN=N(); k < iN; k++) - { - analog_net_t *n = m_nets[k]; - - const nl_double DD_n = (n->Q_Analog() - m_last_V[k]); - const nl_double hn = current_timestep(); - - nl_double DD2 = (DD_n / hn - n->m_DD_n_m_1 / n->m_h_n_m_1) / (hn + n->m_h_n_m_1); - nl_double new_net_timestep; - - n->m_h_n_m_1 = hn; - n->m_DD_n_m_1 = DD_n; - if (nl_math::abs(DD2) > NL_FCONST(1e-30)) // avoid div-by-zero - new_net_timestep = nl_math::sqrt(m_params.m_lte / nl_math::abs(NL_FCONST(0.5)*DD2)); - else - new_net_timestep = m_params.m_max_timestep; - - if (new_net_timestep < new_solver_timestep) - new_solver_timestep = new_net_timestep; - - m_last_V[k] = n->Q_Analog(); - } - if (new_solver_timestep < m_params.m_min_timestep) - new_solver_timestep = m_params.m_min_timestep; - } - //if (new_solver_timestep > 10.0 * hn) - // new_solver_timestep = 10.0 * hn; - return netlist_time::from_double(new_solver_timestep); -} - -template <unsigned m_N, unsigned _storage_N> -ATTR_COLD void matrix_solver_direct_t<m_N, _storage_N>::add_term(int k, terminal_t *term) -{ - if (term->m_otherterm->net().isRailNet()) - { - m_rails_temp[k].add(term, -1, false); - } - else - { - int ot = get_net_idx(&term->m_otherterm->net()); - if (ot>=0) - { - m_terms[k]->add(term, ot, true); - } - /* Should this be allowed ? */ - else // if (ot<0) - { - m_rails_temp[k].add(term, ot, true); - log().fatal("found term with missing othernet {1}\n", term->name()); - } - } -} - - -template <unsigned m_N, unsigned _storage_N> ATTR_COLD void matrix_solver_direct_t<m_N, _storage_N>::vsetup(analog_net_t::list_t &nets) { if (m_dim < nets.size()) log().fatal("Dimension {1} less than {2}", m_dim, nets.size()); - for (unsigned k = 0; k < N(); k++) - { - m_terms[k]->clear(); - m_rails_temp[k].clear(); - } - matrix_solver_t::setup_base(nets); for (unsigned k = 0; k < N(); k++) { m_terms[k]->m_railstart = m_terms[k]->count(); - for (unsigned i = 0; i < m_rails_temp[k].count(); i++) - this->m_terms[k]->add(m_rails_temp[k].terms()[i], m_rails_temp[k].net_other()[i], false); + for (unsigned i = 0; i < m_rails_temp[k]->count(); i++) + this->m_terms[k]->add(m_rails_temp[k]->terms()[i], m_rails_temp[k]->net_other()[i], false); - m_rails_temp[k].clear(); // no longer needed + m_rails_temp[k]->clear(); // no longer needed m_terms[k]->set_pointers(); } + for (unsigned k = 0; k < N(); k++) + pfree(m_rails_temp[k]); // no longer needed + + m_rails_temp.clear(); #if 1 /* Sort in descending order by number of connected matrix voltages. @@ -431,13 +343,13 @@ ATTR_COLD void matrix_solver_direct_t<m_N, _storage_N>::vsetup(analog_net_t::lis * save states */ save(NLNAME(m_last_RHS)); - save(NLNAME(m_last_V)); for (unsigned k = 0; k < N(); k++) { pstring num = pfmt("{1}")(k); save(RHS(k), "RHS" + num); + save(m_terms[k]->m_last_V, "lastV" + num); save(m_terms[k]->go(),"GO" + num, m_terms[k]->count()); save(m_terms[k]->gt(),"GT" + num, m_terms[k]->count()); @@ -447,59 +359,6 @@ ATTR_COLD void matrix_solver_direct_t<m_N, _storage_N>::vsetup(analog_net_t::lis } -template <unsigned m_N, unsigned _storage_N> -void matrix_solver_direct_t<m_N, _storage_N>::build_LE_A() -{ - const unsigned iN = N(); - for (unsigned k = 0; k < iN; k++) - { - for (unsigned i=0; i < iN; i++) - A(k,i) = 0.0; - - const unsigned terms_count = m_terms[k]->count(); - const unsigned railstart = m_terms[k]->m_railstart; - const nl_double * RESTRICT gt = m_terms[k]->gt(); - - { - nl_double akk = 0.0; - for (unsigned i = 0; i < terms_count; i++) - akk += gt[i]; - - A(k,k) = akk; - } - - const nl_double * RESTRICT go = m_terms[k]->go(); - const int * RESTRICT net_other = m_terms[k]->net_other(); - - for (unsigned i = 0; i < railstart; i++) - A(k,net_other[i]) -= go[i]; - } -} - -template <unsigned m_N, unsigned _storage_N> -void matrix_solver_direct_t<m_N, _storage_N>::build_LE_RHS() -{ - const unsigned iN = N(); - for (unsigned k = 0; k < iN; k++) - { - nl_double rhsk_a = 0.0; - nl_double rhsk_b = 0.0; - - const int terms_count = m_terms[k]->count(); - const nl_double * RESTRICT go = m_terms[k]->go(); - const nl_double * RESTRICT Idr = m_terms[k]->Idr(); - const nl_double * const * RESTRICT other_cur_analog = m_terms[k]->other_curanalog(); - - for (int i = 0; i < terms_count; i++) - rhsk_a = rhsk_a + Idr[i]; - - for (int i = m_terms[k]->m_railstart; i < terms_count; i++) - //rhsk = rhsk + go[i] * terms[i]->m_otherterm->net().as_analog().Q_Analog(); - rhsk_b = rhsk_b + go[i] * *other_cur_analog[i]; - - RHS(k) = rhsk_a + rhsk_b; - } -} #if TEST_PARALLEL template <unsigned m_N, unsigned _storage_N> @@ -587,9 +446,9 @@ void matrix_solver_direct_t<m_N, _storage_N>::LE_solve() { #if TEST_PARALLEL const unsigned eb = m_terms[i]->m_nzbd.size(); - if (eb > 16) + if (eb > 0) { - printf("here %d\n", eb); + //printf("here %d\n", eb); unsigned chunks = (eb) / (num_thr + 1); for (int p=0; p < num_thr + 1; p++) { @@ -617,10 +476,10 @@ void matrix_solver_direct_t<m_N, _storage_N>::LE_solve() /* Eliminate column i from row j */ - for (auto j : nzbd) + for (auto & j : nzbd) { const nl_double f1 = - A(j,i) * f; - for (auto k : nzrd) + for (auto & k : nzrd) A(j,k) += A(i,k) * f1; //RHS(j) += RHS(i) * f1; } @@ -632,10 +491,10 @@ void matrix_solver_direct_t<m_N, _storage_N>::LE_solve() /* Eliminate column i from row j */ - for (auto j : nzbd) + for (auto & j : nzbd) { const nl_double f1 = - A(j,i) * f; - for (auto k : nzrd) + for (auto & k : nzrd) A(j,k) += A(i,k) * f1; //RHS(j) += RHS(i) * f1; } @@ -684,35 +543,6 @@ void matrix_solver_direct_t<m_N, _storage_N>::LE_back_subst( template <unsigned m_N, unsigned _storage_N> -template <typename T> -T matrix_solver_direct_t<m_N, _storage_N>::delta( - const T * RESTRICT V) -{ - /* FIXME: Ideally we should also include currents (RHS) here. This would - * need a revaluation of the right hand side after voltages have been updated - * and thus belong into a different calculation. This applies to all solvers. - */ - - const unsigned iN = this->N(); - T cerr = 0; - for (unsigned i = 0; i < iN; i++) - cerr = std::fmax(cerr, nl_math::abs(V[i] - (T) this->m_nets[i]->m_cur_Analog)); - return cerr; -} - -template <unsigned m_N, unsigned _storage_N> -template <typename T> -void matrix_solver_direct_t<m_N, _storage_N>::store( - const T * RESTRICT V) -{ - for (unsigned i = 0, iN=N(); i < iN; i++) - { - this->m_nets[i]->m_cur_Analog = V[i]; - } -} - - -template <unsigned m_N, unsigned _storage_N> int matrix_solver_direct_t<m_N, _storage_N>::solve_non_dynamic(ATTR_UNUSED const bool newton_raphson) { nl_double new_V[_storage_N]; // = { 0.0 }; @@ -738,8 +568,8 @@ int matrix_solver_direct_t<m_N, _storage_N>::solve_non_dynamic(ATTR_UNUSED const template <unsigned m_N, unsigned _storage_N> inline int matrix_solver_direct_t<m_N, _storage_N>::vsolve_non_dynamic(const bool newton_raphson) { - this->build_LE_A(); - this->build_LE_RHS(); + this->build_LE_A(*this); + this->build_LE_RHS(*this); for (unsigned i=0, iN=N(); i < iN; i++) m_last_RHS[i] = RHS(i); @@ -753,15 +583,12 @@ matrix_solver_direct_t<m_N, _storage_N>::matrix_solver_direct_t(const solver_par : matrix_solver_t(GAUSSIAN_ELIMINATION, params) , m_dim(size) { - m_rails_temp = palloc_array(terms_t, N()); #if (NL_USE_DYNAMIC_ALLOCATION) m_A = palloc_array(nl_ext_double, N() * m_pitch); #endif for (unsigned k = 0; k < N(); k++) { - m_terms[k] = palloc(terms_t); m_last_RHS[k] = 0.0; - m_last_V[k] = 0.0; } #if TEST_PARALLEL thr_initialize(); @@ -773,15 +600,12 @@ matrix_solver_direct_t<m_N, _storage_N>::matrix_solver_direct_t(const eSolverTyp : matrix_solver_t(type, params) , m_dim(size) { - m_rails_temp = palloc_array(terms_t, N()); #if (NL_USE_DYNAMIC_ALLOCATION) m_A = palloc_array(nl_ext_double, N() * m_pitch); #endif for (unsigned k = 0; k < N(); k++) { - m_terms[k] = palloc(terms_t); m_last_RHS[k] = 0.0; - m_last_V[k] = 0.0; } #if TEST_PARALLEL thr_initialize(); diff --git a/src/lib/netlist/solver/nld_ms_direct1.h b/src/lib/netlist/solver/nld_ms_direct1.h index ab19c92a4cb..abf9518bfbe 100644 --- a/src/lib/netlist/solver/nld_ms_direct1.h +++ b/src/lib/netlist/solver/nld_ms_direct1.h @@ -30,8 +30,8 @@ public: inline int matrix_solver_direct1_t::vsolve_non_dynamic(ATTR_UNUSED const bool newton_raphson) { - this->build_LE_A(); - this->build_LE_RHS(); + this->build_LE_A(*this); + this->build_LE_RHS(*this); //NL_VERBOSE_OUT(("{1} {2}\n", new_val, m_RHS[0] / m_A[0][0]); nl_double new_val[1] = { RHS(0) / A(0,0) }; diff --git a/src/lib/netlist/solver/nld_ms_direct2.h b/src/lib/netlist/solver/nld_ms_direct2.h index 2488e573bf2..f71133d95f0 100644 --- a/src/lib/netlist/solver/nld_ms_direct2.h +++ b/src/lib/netlist/solver/nld_ms_direct2.h @@ -30,8 +30,8 @@ public: inline int matrix_solver_direct2_t::vsolve_non_dynamic(ATTR_UNUSED const bool newton_raphson) { - build_LE_A(); - build_LE_RHS(); + build_LE_A(*this); + build_LE_RHS(*this); const nl_double a = A(0,0); const nl_double b = A(0,1); diff --git a/src/lib/netlist/solver/nld_ms_direct_lu.h b/src/lib/netlist/solver/nld_ms_direct_lu.h index 757970e5c07..e6ccf819653 100644 --- a/src/lib/netlist/solver/nld_ms_direct_lu.h +++ b/src/lib/netlist/solver/nld_ms_direct_lu.h @@ -144,7 +144,6 @@ protected: ATTR_ALIGN nl_double m_last_RHS[_storage_N]; // right hand side - contains currents ATTR_ALIGN nl_double m_last_V[_storage_N]; - terms_t **m_terms; terms_t *m_rails_temp; private: @@ -161,11 +160,6 @@ private: template <unsigned m_N, unsigned _storage_N> matrix_solver_direct_t<m_N, _storage_N>::~matrix_solver_direct_t() { - for (unsigned k = 0; k < N(); k++) - { - pfree(m_terms[k]); - } - pfree_array(m_terms); pfree_array(m_rails_temp); } @@ -608,14 +602,11 @@ matrix_solver_direct_t<m_N, _storage_N>::matrix_solver_direct_t(const solver_par , m_dim(size) , m_lp_fact(0) { - m_terms = palloc_array(terms_t *, N()); m_rails_temp = palloc_array(terms_t, N()); for (unsigned k = 0; k < N(); k++) { - m_terms[k] = palloc(terms_t); m_last_RHS[k] = 0.0; - m_last_V[k] = 0.0; } } @@ -625,14 +616,12 @@ matrix_solver_direct_t<m_N, _storage_N>::matrix_solver_direct_t(const eSolverTyp , m_dim(size) , m_lp_fact(0) { - m_terms = palloc_array(terms_t *, N()); m_rails_temp = palloc_array(terms_t, N()); for (unsigned k = 0; k < N(); k++) { m_terms[k] = palloc(terms_t); m_last_RHS[k] = 0.0; - m_last_V[k] = 0.0; } } diff --git a/src/lib/netlist/solver/nld_ms_sm.h b/src/lib/netlist/solver/nld_ms_sm.h index 3dd440d4b15..bff8197127c 100644 --- a/src/lib/netlist/solver/nld_ms_sm.h +++ b/src/lib/netlist/solver/nld_ms_sm.h @@ -48,6 +48,8 @@ NETLIB_NAMESPACE_DEVICES_START() template <unsigned m_N, unsigned _storage_N> class matrix_solver_sm_t: public matrix_solver_t { + friend class matrix_solver_t; + public: matrix_solver_sm_t(const solver_parameters_t *params, const int size); @@ -59,26 +61,16 @@ public: virtual void reset() override { matrix_solver_t::reset(); } protected: - virtual void add_term(int net_idx, terminal_t *term) override; virtual int vsolve_non_dynamic(const bool newton_raphson) override; int solve_non_dynamic(const bool newton_raphson); inline unsigned N() const { if (m_N == 0) return m_dim; else return m_N; } - void build_LE_A(); - void build_LE_RHS(); void LE_invert(); template <typename T> void LE_compute_x(T * RESTRICT x); - template <typename T> - T delta(const T * RESTRICT V); - - template <typename T> - void store(const T * RESTRICT V); - - virtual netlist_time compute_next_timestep() override; template <typename T1, typename T2> inline nl_ext_double &A(const T1 &r, const T2 &c) { return m_A[r][c]; } @@ -96,10 +88,6 @@ protected: inline nl_ext_double &lAinv(const T1 &r, const T2 &c) { return m_lAinv[r][c]; } ATTR_ALIGN nl_double m_last_RHS[_storage_N]; // right hand side - contains currents - ATTR_ALIGN nl_double m_last_V[_storage_N]; - - terms_t * m_terms[_storage_N]; - terms_t *m_rails_temp; private: static const std::size_t m_pitch = ((( _storage_N) + 7) / 8) * 8; @@ -124,102 +112,26 @@ private: template <unsigned m_N, unsigned _storage_N> matrix_solver_sm_t<m_N, _storage_N>::~matrix_solver_sm_t() { - for (unsigned k = 0; k < N(); k++) - { - pfree(m_terms[k]); - } - pfree_array(m_rails_temp); #if (NL_USE_DYNAMIC_ALLOCATION) pfree_array(m_A); #endif } template <unsigned m_N, unsigned _storage_N> -netlist_time matrix_solver_sm_t<m_N, _storage_N>::compute_next_timestep() -{ - nl_double new_solver_timestep = m_params.m_max_timestep; - - if (m_params.m_dynamic) - { - /* - * FIXME: We should extend the logic to use either all nets or - * only output nets. - */ - for (unsigned k = 0, iN=N(); k < iN; k++) - { - analog_net_t *n = m_nets[k]; - - const nl_double DD_n = (n->Q_Analog() - m_last_V[k]); - const nl_double hn = current_timestep(); - - nl_double DD2 = (DD_n / hn - n->m_DD_n_m_1 / n->m_h_n_m_1) / (hn + n->m_h_n_m_1); - nl_double new_net_timestep; - - n->m_h_n_m_1 = hn; - n->m_DD_n_m_1 = DD_n; - if (nl_math::abs(DD2) > NL_FCONST(1e-30)) // avoid div-by-zero - new_net_timestep = nl_math::sqrt(m_params.m_lte / nl_math::abs(NL_FCONST(0.5)*DD2)); - else - new_net_timestep = m_params.m_max_timestep; - - if (new_net_timestep < new_solver_timestep) - new_solver_timestep = new_net_timestep; - - m_last_V[k] = n->Q_Analog(); - } - if (new_solver_timestep < m_params.m_min_timestep) - new_solver_timestep = m_params.m_min_timestep; - } - //if (new_solver_timestep > 10.0 * hn) - // new_solver_timestep = 10.0 * hn; - return netlist_time::from_double(new_solver_timestep); -} - -template <unsigned m_N, unsigned _storage_N> -ATTR_COLD void matrix_solver_sm_t<m_N, _storage_N>::add_term(int k, terminal_t *term) -{ - if (term->m_otherterm->net().isRailNet()) - { - m_rails_temp[k].add(term, -1, false); - } - else - { - int ot = get_net_idx(&term->m_otherterm->net()); - if (ot>=0) - { - m_terms[k]->add(term, ot, true); - } - /* Should this be allowed ? */ - else // if (ot<0) - { - m_rails_temp[k].add(term, ot, true); - log().fatal("found term with missing othernet {1}\n", term->name()); - } - } -} - - -template <unsigned m_N, unsigned _storage_N> ATTR_COLD void matrix_solver_sm_t<m_N, _storage_N>::vsetup(analog_net_t::list_t &nets) { if (m_dim < nets.size()) log().fatal("Dimension {1} less than {2}", m_dim, nets.size()); - for (unsigned k = 0; k < N(); k++) - { - m_terms[k]->clear(); - m_rails_temp[k].clear(); - } - matrix_solver_t::setup_base(nets); for (unsigned k = 0; k < N(); k++) { m_terms[k]->m_railstart = m_terms[k]->count(); - for (unsigned i = 0; i < m_rails_temp[k].count(); i++) - this->m_terms[k]->add(m_rails_temp[k].terms()[i], m_rails_temp[k].net_other()[i], false); + for (unsigned i = 0; i < m_rails_temp[k]->count(); i++) + this->m_terms[k]->add(m_rails_temp[k]->terms()[i], m_rails_temp[k]->net_other()[i], false); - m_rails_temp[k].clear(); // no longer needed + m_rails_temp[k]->clear(); // no longer needed m_terms[k]->set_pointers(); } @@ -315,13 +227,13 @@ ATTR_COLD void matrix_solver_sm_t<m_N, _storage_N>::vsetup(analog_net_t::list_t * save states */ save(NLNAME(m_last_RHS)); - save(NLNAME(m_last_V)); for (unsigned k = 0; k < N(); k++) { pstring num = pfmt("{1}")(k); save(RHS(k), "RHS" + num); + save(m_terms[k]->m_last_V, "lastV" + num); save(m_terms[k]->go(),"GO" + num, m_terms[k]->count()); save(m_terms[k]->gt(),"GT" + num, m_terms[k]->count()); @@ -331,59 +243,6 @@ ATTR_COLD void matrix_solver_sm_t<m_N, _storage_N>::vsetup(analog_net_t::list_t } -template <unsigned m_N, unsigned _storage_N> -void matrix_solver_sm_t<m_N, _storage_N>::build_LE_A() -{ - const unsigned iN = N(); - for (unsigned k = 0; k < iN; k++) - { - for (unsigned i=0; i < iN; i++) - A(k,i) = 0.0; - - const unsigned terms_count = m_terms[k]->count(); - const unsigned railstart = m_terms[k]->m_railstart; - const nl_double * RESTRICT gt = m_terms[k]->gt(); - - { - nl_double akk = 0.0; - for (unsigned i = 0; i < terms_count; i++) - akk += gt[i]; - - A(k,k) = akk; - } - - const nl_double * RESTRICT go = m_terms[k]->go(); - const int * RESTRICT net_other = m_terms[k]->net_other(); - - for (unsigned i = 0; i < railstart; i++) - A(k,net_other[i]) -= go[i]; - } -} - -template <unsigned m_N, unsigned _storage_N> -void matrix_solver_sm_t<m_N, _storage_N>::build_LE_RHS() -{ - const unsigned iN = N(); - for (unsigned k = 0; k < iN; k++) - { - nl_double rhsk_a = 0.0; - nl_double rhsk_b = 0.0; - - const int terms_count = m_terms[k]->count(); - const nl_double * RESTRICT go = m_terms[k]->go(); - const nl_double * RESTRICT Idr = m_terms[k]->Idr(); - const nl_double * const * RESTRICT other_cur_analog = m_terms[k]->other_curanalog(); - - for (int i = 0; i < terms_count; i++) - rhsk_a = rhsk_a + Idr[i]; - - for (int i = m_terms[k]->m_railstart; i < terms_count; i++) - //rhsk = rhsk + go[i] * terms[i]->m_otherterm->net().as_analog().Q_Analog(); - rhsk_b = rhsk_b + go[i] * *other_cur_analog[i]; - - RHS(k) = rhsk_a + rhsk_b; - } -} template <unsigned m_N, unsigned _storage_N> void matrix_solver_sm_t<m_N, _storage_N>::LE_invert() @@ -469,35 +328,6 @@ void matrix_solver_sm_t<m_N, _storage_N>::LE_compute_x( template <unsigned m_N, unsigned _storage_N> -template <typename T> -T matrix_solver_sm_t<m_N, _storage_N>::delta( - const T * RESTRICT V) -{ - /* FIXME: Ideally we should also include currents (RHS) here. This would - * need a revaluation of the right hand side after voltages have been updated - * and thus belong into a different calculation. This applies to all solvers. - */ - - const unsigned iN = this->N(); - T cerr = 0; - for (unsigned i = 0; i < iN; i++) - cerr = std::fmax(cerr, nl_math::abs(V[i] - (T) this->m_nets[i]->m_cur_Analog)); - return cerr; -} - -template <unsigned m_N, unsigned _storage_N> -template <typename T> -void matrix_solver_sm_t<m_N, _storage_N>::store( - const T * RESTRICT V) -{ - for (unsigned i = 0, iN=N(); i < iN; i++) - { - this->m_nets[i]->m_cur_Analog = V[i]; - } -} - - -template <unsigned m_N, unsigned _storage_N> int matrix_solver_sm_t<m_N, _storage_N>::solve_non_dynamic(ATTR_UNUSED const bool newton_raphson) { static const bool incremental = true; @@ -590,8 +420,8 @@ int matrix_solver_sm_t<m_N, _storage_N>::solve_non_dynamic(ATTR_UNUSED const boo template <unsigned m_N, unsigned _storage_N> inline int matrix_solver_sm_t<m_N, _storage_N>::vsolve_non_dynamic(const bool newton_raphson) { - this->build_LE_A(); - this->build_LE_RHS(); + this->build_LE_A(*this); + this->build_LE_RHS(*this); for (unsigned i=0, iN=N(); i < iN; i++) m_last_RHS[i] = RHS(i); @@ -605,15 +435,12 @@ matrix_solver_sm_t<m_N, _storage_N>::matrix_solver_sm_t(const solver_parameters_ : matrix_solver_t(GAUSSIAN_ELIMINATION, params) , m_dim(size) { - m_rails_temp = palloc_array(terms_t, N()); #if (NL_USE_DYNAMIC_ALLOCATION) m_A = palloc_array(nl_ext_double, N() * m_pitch); #endif for (unsigned k = 0; k < N(); k++) { - m_terms[k] = palloc(terms_t); m_last_RHS[k] = 0.0; - m_last_V[k] = 0.0; } } @@ -622,15 +449,12 @@ matrix_solver_sm_t<m_N, _storage_N>::matrix_solver_sm_t(const eSolverType type, : matrix_solver_t(type, params) , m_dim(size) { - m_rails_temp = palloc_array(terms_t, N()); #if (NL_USE_DYNAMIC_ALLOCATION) m_A = palloc_array(nl_ext_double, N() * m_pitch); #endif for (unsigned k = 0; k < N(); k++) { - m_terms[k] = palloc(terms_t); m_last_RHS[k] = 0.0; - m_last_V[k] = 0.0; } } diff --git a/src/lib/netlist/solver/nld_ms_sor_mat.h b/src/lib/netlist/solver/nld_ms_sor_mat.h index a4fce8dec2a..10192717a90 100644 --- a/src/lib/netlist/solver/nld_ms_sor_mat.h +++ b/src/lib/netlist/solver/nld_ms_sor_mat.h @@ -128,8 +128,8 @@ int matrix_solver_SOR_mat_t<m_N, _storage_N>::vsolve_non_dynamic(const bool newt int resched_cnt = 0; - this->build_LE_A(); - this->build_LE_RHS(); + this->build_LE_A(*this); + this->build_LE_RHS(*this); #if 0 static int ws_cnt = 0; diff --git a/src/lib/netlist/solver/nld_ms_w.h b/src/lib/netlist/solver/nld_ms_w.h index 154f618a697..aa6f7104c6c 100644 --- a/src/lib/netlist/solver/nld_ms_w.h +++ b/src/lib/netlist/solver/nld_ms_w.h @@ -55,6 +55,7 @@ NETLIB_NAMESPACE_DEVICES_START() template <unsigned m_N, unsigned _storage_N> class matrix_solver_w_t: public matrix_solver_t { + friend class matrix_solver_t; public: matrix_solver_w_t(const solver_parameters_t *params, const int size); @@ -66,26 +67,16 @@ public: virtual void reset() override { matrix_solver_t::reset(); } protected: - virtual void add_term(int net_idx, terminal_t *term) override; virtual int vsolve_non_dynamic(const bool newton_raphson) override; int solve_non_dynamic(const bool newton_raphson); inline unsigned N() const { if (m_N == 0) return m_dim; else return m_N; } - void build_LE_A(); - void build_LE_RHS(); void LE_invert(); template <typename T> void LE_compute_x(T * RESTRICT x); - template <typename T> - T delta(const T * RESTRICT V); - - template <typename T> - void store(const T * RESTRICT V); - - virtual netlist_time compute_next_timestep() override; template <typename T1, typename T2> inline nl_ext_double &A(const T1 &r, const T2 &c) { return m_A[r][c]; } @@ -103,10 +94,6 @@ protected: inline nl_ext_double &lA(const T1 &r, const T2 &c) { return m_lA[r][c]; } ATTR_ALIGN nl_double m_last_RHS[_storage_N]; // right hand side - contains currents - ATTR_ALIGN nl_double m_last_V[_storage_N]; - - terms_t * m_terms[_storage_N]; - terms_t *m_rails_temp; private: static const std::size_t m_pitch = ((( _storage_N) + 7) / 8) * 8; @@ -138,102 +125,26 @@ private: template <unsigned m_N, unsigned _storage_N> matrix_solver_w_t<m_N, _storage_N>::~matrix_solver_w_t() { - for (unsigned k = 0; k < N(); k++) - { - pfree(m_terms[k]); - } - pfree_array(m_rails_temp); #if (NL_USE_DYNAMIC_ALLOCATION) pfree_array(m_A); #endif } template <unsigned m_N, unsigned _storage_N> -netlist_time matrix_solver_w_t<m_N, _storage_N>::compute_next_timestep() -{ - nl_double new_solver_timestep = m_params.m_max_timestep; - - if (m_params.m_dynamic) - { - /* - * FIXME: We should extend the logic to use either all nets or - * only output nets. - */ - for (unsigned k = 0, iN=N(); k < iN; k++) - { - analog_net_t *n = m_nets[k]; - - const nl_double DD_n = (n->Q_Analog() - m_last_V[k]); - const nl_double hn = current_timestep(); - - nl_double DD2 = (DD_n / hn - n->m_DD_n_m_1 / n->m_h_n_m_1) / (hn + n->m_h_n_m_1); - nl_double new_net_timestep; - - n->m_h_n_m_1 = hn; - n->m_DD_n_m_1 = DD_n; - if (nl_math::abs(DD2) > NL_FCONST(1e-30)) // avoid div-by-zero - new_net_timestep = nl_math::sqrt(m_params.m_lte / nl_math::abs(NL_FCONST(0.5)*DD2)); - else - new_net_timestep = m_params.m_max_timestep; - - if (new_net_timestep < new_solver_timestep) - new_solver_timestep = new_net_timestep; - - m_last_V[k] = n->Q_Analog(); - } - if (new_solver_timestep < m_params.m_min_timestep) - new_solver_timestep = m_params.m_min_timestep; - } - //if (new_solver_timestep > 10.0 * hn) - // new_solver_timestep = 10.0 * hn; - return netlist_time::from_double(new_solver_timestep); -} - -template <unsigned m_N, unsigned _storage_N> -ATTR_COLD void matrix_solver_w_t<m_N, _storage_N>::add_term(int k, terminal_t *term) -{ - if (term->m_otherterm->net().isRailNet()) - { - m_rails_temp[k].add(term, -1, false); - } - else - { - int ot = get_net_idx(&term->m_otherterm->net()); - if (ot>=0) - { - m_terms[k]->add(term, ot, true); - } - /* Should this be allowed ? */ - else // if (ot<0) - { - m_rails_temp[k].add(term, ot, true); - log().fatal("found term with missing othernet {1}\n", term->name()); - } - } -} - - -template <unsigned m_N, unsigned _storage_N> ATTR_COLD void matrix_solver_w_t<m_N, _storage_N>::vsetup(analog_net_t::list_t &nets) { if (m_dim < nets.size()) log().fatal("Dimension {1} less than {2}", m_dim, nets.size()); - for (unsigned k = 0; k < N(); k++) - { - m_terms[k]->clear(); - m_rails_temp[k].clear(); - } - matrix_solver_t::setup_base(nets); for (unsigned k = 0; k < N(); k++) { m_terms[k]->m_railstart = m_terms[k]->count(); - for (unsigned i = 0; i < m_rails_temp[k].count(); i++) - this->m_terms[k]->add(m_rails_temp[k].terms()[i], m_rails_temp[k].net_other()[i], false); + for (unsigned i = 0; i < m_rails_temp[k]->count(); i++) + this->m_terms[k]->add(m_rails_temp[k]->terms()[i], m_rails_temp[k]->net_other()[i], false); - m_rails_temp[k].clear(); // no longer needed + m_rails_temp[k]->clear(); // no longer needed m_terms[k]->set_pointers(); } @@ -329,13 +240,13 @@ ATTR_COLD void matrix_solver_w_t<m_N, _storage_N>::vsetup(analog_net_t::list_t & * save states */ save(NLNAME(m_last_RHS)); - save(NLNAME(m_last_V)); for (unsigned k = 0; k < N(); k++) { pstring num = pfmt("{1}")(k); save(RHS(k), "RHS" + num); + save(m_terms[k]->m_last_V, "lastV" + num); save(m_terms[k]->go(),"GO" + num, m_terms[k]->count()); save(m_terms[k]->gt(),"GT" + num, m_terms[k]->count()); @@ -345,59 +256,6 @@ ATTR_COLD void matrix_solver_w_t<m_N, _storage_N>::vsetup(analog_net_t::list_t & } -template <unsigned m_N, unsigned _storage_N> -void matrix_solver_w_t<m_N, _storage_N>::build_LE_A() -{ - const unsigned iN = N(); - for (unsigned k = 0; k < iN; k++) - { - for (unsigned i=0; i < iN; i++) - A(k,i) = 0.0; - - const unsigned terms_count = m_terms[k]->count(); - const unsigned railstart = m_terms[k]->m_railstart; - const nl_double * RESTRICT gt = m_terms[k]->gt(); - - { - nl_double akk = 0.0; - for (unsigned i = 0; i < terms_count; i++) - akk += gt[i]; - - A(k,k) = akk; - } - - const nl_double * RESTRICT go = m_terms[k]->go(); - const int * RESTRICT net_other = m_terms[k]->net_other(); - - for (unsigned i = 0; i < railstart; i++) - A(k,net_other[i]) -= go[i]; - } -} - -template <unsigned m_N, unsigned _storage_N> -void matrix_solver_w_t<m_N, _storage_N>::build_LE_RHS() -{ - const unsigned iN = N(); - for (unsigned k = 0; k < iN; k++) - { - nl_double rhsk_a = 0.0; - nl_double rhsk_b = 0.0; - - const int terms_count = m_terms[k]->count(); - const nl_double * RESTRICT go = m_terms[k]->go(); - const nl_double * RESTRICT Idr = m_terms[k]->Idr(); - const nl_double * const * RESTRICT other_cur_analog = m_terms[k]->other_curanalog(); - - for (int i = 0; i < terms_count; i++) - rhsk_a = rhsk_a + Idr[i]; - - for (int i = m_terms[k]->m_railstart; i < terms_count; i++) - //rhsk = rhsk + go[i] * terms[i]->m_otherterm->net().as_analog().Q_Analog(); - rhsk_b = rhsk_b + go[i] * *other_cur_analog[i]; - - RHS(k) = rhsk_a + rhsk_b; - } -} template <unsigned m_N, unsigned _storage_N> void matrix_solver_w_t<m_N, _storage_N>::LE_invert() @@ -482,34 +340,6 @@ void matrix_solver_w_t<m_N, _storage_N>::LE_compute_x( template <unsigned m_N, unsigned _storage_N> -template <typename T> -T matrix_solver_w_t<m_N, _storage_N>::delta( - const T * RESTRICT V) -{ - /* FIXME: Ideally we should also include currents (RHS) here. This would - * need a revaluation of the right hand side after voltages have been updated - * and thus belong into a different calculation. This applies to all solvers. - */ - - const unsigned iN = this->N(); - T cerr = 0; - for (unsigned i = 0; i < iN; i++) - cerr = std::fmax(cerr, nl_math::abs(V[i] - (T) this->m_nets[i]->m_cur_Analog)); - return cerr; -} - -template <unsigned m_N, unsigned _storage_N> -template <typename T> -void matrix_solver_w_t<m_N, _storage_N>::store( - const T * RESTRICT V) -{ - for (unsigned i = 0, iN=N(); i < iN; i++) - { - this->m_nets[i]->m_cur_Analog = V[i]; - } -} - -template <unsigned m_N, unsigned _storage_N> int matrix_solver_w_t<m_N, _storage_N>::solve_non_dynamic(ATTR_UNUSED const bool newton_raphson) { const auto iN = N(); @@ -652,8 +482,8 @@ int matrix_solver_w_t<m_N, _storage_N>::solve_non_dynamic(ATTR_UNUSED const bool template <unsigned m_N, unsigned _storage_N> inline int matrix_solver_w_t<m_N, _storage_N>::vsolve_non_dynamic(const bool newton_raphson) { - this->build_LE_A(); - this->build_LE_RHS(); + this->build_LE_A(*this); + this->build_LE_RHS(*this); for (unsigned i=0, iN=N(); i < iN; i++) m_last_RHS[i] = RHS(i); @@ -668,15 +498,12 @@ matrix_solver_w_t<m_N, _storage_N>::matrix_solver_w_t(const solver_parameters_t ,m_cnt(0) , m_dim(size) { - m_rails_temp = palloc_array(terms_t, N()); #if (NL_USE_DYNAMIC_ALLOCATION) m_A = palloc_array(nl_ext_double, N() * m_pitch); #endif for (unsigned k = 0; k < N(); k++) { - m_terms[k] = palloc(terms_t); m_last_RHS[k] = 0.0; - m_last_V[k] = 0.0; } } @@ -686,12 +513,9 @@ matrix_solver_w_t<m_N, _storage_N>::matrix_solver_w_t(const eSolverType type, co ,m_cnt(0) , m_dim(size) { - m_rails_temp = palloc_array(terms_t, N()); for (unsigned k = 0; k < N(); k++) { - m_terms[k] = palloc(terms_t); m_last_RHS[k] = 0.0; - m_last_V[k] = 0.0; } } diff --git a/src/lib/netlist/solver/nld_solver.cpp b/src/lib/netlist/solver/nld_solver.cpp index ed011ff254d..5dd0af53313 100644 --- a/src/lib/netlist/solver/nld_solver.cpp +++ b/src/lib/netlist/solver/nld_solver.cpp @@ -108,6 +108,11 @@ ATTR_COLD matrix_solver_t::matrix_solver_t(const eSolverType type, const solver_ ATTR_COLD matrix_solver_t::~matrix_solver_t() { m_inps.clear_and_free(); + for (unsigned k = 0; k < m_terms.size(); k++) + { + pfree(m_terms[k]); + } + } ATTR_COLD void matrix_solver_t::setup_base(analog_net_t::list_t &nets) @@ -115,9 +120,14 @@ ATTR_COLD void matrix_solver_t::setup_base(analog_net_t::list_t &nets) log().debug("New solver setup\n"); m_nets.clear(); + m_terms.clear(); for (auto & net : nets) + { m_nets.push_back(net); + m_terms.push_back(palloc(terms_t)); + m_rails_temp.push_back(palloc(terms_t)); + } for (std::size_t k = 0; k < nets.size(); k++) { @@ -168,6 +178,9 @@ ATTR_COLD void matrix_solver_t::setup_base(analog_net_t::list_t &nets) if (net_proxy_output == NULL) { + //net_proxy_output = palloc(analog_output_t(*this, + // this->name() + "." + pfmt("m{1}")(m_inps.size()))); + net_proxy_output = palloc(analog_output_t); net_proxy_output->init_object(*this, this->name() + "." + pfmt("m{1}")(m_inps.size())); m_inps.push_back(net_proxy_output); @@ -308,6 +321,70 @@ ATTR_COLD int matrix_solver_t::get_net_idx(net_t *net) return -1; } +ATTR_COLD void matrix_solver_t::add_term(int k, terminal_t *term) +{ + if (term->m_otherterm->net().isRailNet()) + { + m_rails_temp[k]->add(term, -1, false); + } + else + { + int ot = get_net_idx(&term->m_otherterm->net()); + if (ot>=0) + { + m_terms[k]->add(term, ot, true); + } + /* Should this be allowed ? */ + else // if (ot<0) + { + m_rails_temp[k]->add(term, ot, true); + log().fatal("found term with missing othernet {1}\n", term->name()); + } + } +} + +netlist_time matrix_solver_t::compute_next_timestep() +{ + nl_double new_solver_timestep = m_params.m_max_timestep; + + if (m_params.m_dynamic) + { + /* + * FIXME: We should extend the logic to use either all nets or + * only output nets. + */ + for (unsigned k = 0, iN=m_terms.size(); k < iN; k++) + { + analog_net_t *n = m_nets[k]; + + const nl_double DD_n = (n->Q_Analog() - m_terms[k]->m_last_V); + const nl_double hn = current_timestep(); + + nl_double DD2 = (DD_n / hn - n->m_DD_n_m_1 / n->m_h_n_m_1) / (hn + n->m_h_n_m_1); + nl_double new_net_timestep; + + n->m_h_n_m_1 = hn; + n->m_DD_n_m_1 = DD_n; + if (nl_math::abs(DD2) > NL_FCONST(1e-30)) // avoid div-by-zero + new_net_timestep = nl_math::sqrt(m_params.m_lte / nl_math::abs(NL_FCONST(0.5)*DD2)); + else + new_net_timestep = m_params.m_max_timestep; + + if (new_net_timestep < new_solver_timestep) + new_solver_timestep = new_net_timestep; + + m_terms[k]->m_last_V = n->Q_Analog(); + } + if (new_solver_timestep < m_params.m_min_timestep) + new_solver_timestep = m_params.m_min_timestep; + } + //if (new_solver_timestep > 10.0 * hn) + // new_solver_timestep = 10.0 * hn; + return netlist_time::from_double(new_solver_timestep); +} + + + void matrix_solver_t::log_stats() { if (this->m_stat_calculations != 0 && this->m_stat_vsolver_calls && this->m_params.m_log_stats) diff --git a/src/lib/util/chdcodec.cpp b/src/lib/util/chdcodec.cpp index be1d2022cd7..ba1840489d6 100644 --- a/src/lib/util/chdcodec.cpp +++ b/src/lib/util/chdcodec.cpp @@ -1128,20 +1128,35 @@ chd_lzma_decompressor::chd_lzma_decompressor(chd_file &chd, UINT32 hunkbytes, bo // construct the decoder LzmaDec_Construct(&m_decoder); + // FIXME: this code is written in a way that makes it impossible to safely upgrade the LZMA SDK + // This code assumes that the current version of the encoder imposes the same requirements on the + // decoder as the encoder used to produce the file. This is not necessarily true. The format + // needs to be changed so the encoder properties are written to the file. + // configure the properties like the compressor did CLzmaEncProps encoder_props; chd_lzma_compressor::configure_properties(encoder_props, hunkbytes); // convert to decoder properties - CLzmaProps decoder_props; - decoder_props.lc = encoder_props.lc; - decoder_props.lp = encoder_props.lp; - decoder_props.pb = encoder_props.pb; - decoder_props.dicSize = encoder_props.dictSize; + CLzmaEncHandle enc = LzmaEnc_Create(&m_allocator); + if (!enc) + throw CHDERR_DECOMPRESSION_ERROR; + if (LzmaEnc_SetProps(enc, &encoder_props) != SZ_OK) + { + LzmaEnc_Destroy(enc, &m_allocator, &m_allocator); + throw CHDERR_DECOMPRESSION_ERROR; + } + Byte decoder_props[LZMA_PROPS_SIZE]; + SizeT props_size = sizeof(decoder_props); + if (LzmaEnc_WriteProperties(enc, decoder_props, &props_size) != SZ_OK) + { + LzmaEnc_Destroy(enc, &m_allocator, &m_allocator); + throw CHDERR_DECOMPRESSION_ERROR; + } + LzmaEnc_Destroy(enc, &m_allocator, &m_allocator); // do memory allocations - SRes res = LzmaDec_Allocate_MAME(&m_decoder, &decoder_props, &m_allocator); - if (res != SZ_OK) + if (LzmaDec_Allocate(&m_decoder, decoder_props, LZMA_PROPS_SIZE, &m_allocator) != SZ_OK) throw CHDERR_DECOMPRESSION_ERROR; } diff --git a/src/lib/util/un7z.cpp b/src/lib/util/un7z.cpp index 5822479c4f0..a2fbf380d35 100644 --- a/src/lib/util/un7z.cpp +++ b/src/lib/util/un7z.cpp @@ -404,25 +404,24 @@ int m7z_file_impl::search( bool matchname, bool partialpath) { - for ( ; i < m_db.db.NumFiles; i++) + for ( ; i < m_db.NumFiles; i++) { - const CSzFileItem &f(m_db.db.Files[i]); - make_utf8_name(i); - const std::uint64_t size(f.Size); - const std::uint32_t crc(f.Crc); - const bool crcmatch(crc == search_crc); + bool const is_dir(SzArEx_IsDir(&m_db, i)); + const std::uint64_t size(SzArEx_GetFileSize(&m_db, i)); + const std::uint32_t crc(m_db.CRCs.Vals[i]); + const bool crcmatch(SzBitArray_Check(m_db.CRCs.Defs, i) && (crc == search_crc)); auto const partialoffset(m_utf8_buf.size() - 1 - search_filename.length()); bool const partialpossible((m_utf8_buf.size() > (search_filename.length() + 1)) && (m_utf8_buf[partialoffset - 1] == '/')); const bool namematch( !core_stricmp(search_filename.c_str(), &m_utf8_buf[0]) || (partialpath && partialpossible && !core_stricmp(search_filename.c_str(), &m_utf8_buf[partialoffset]))); - const bool found = ((!matchcrc && !matchname) || !f.IsDir) && (!matchcrc || crcmatch) && (!matchname || namematch); + const bool found = ((!matchcrc && !matchname) || !is_dir) && (!matchcrc || crcmatch) && (!matchname || namematch); if (found) { m_curr_file_idx = i; - m_curr_is_dir = bool(f.IsDir); + m_curr_is_dir = is_dir; m_curr_name = &m_utf8_buf[0]; m_curr_length = size; m_curr_crc = crc; diff --git a/src/lib/util/unzip.cpp b/src/lib/util/unzip.cpp index 6a9e407252d..181a192d002 100644 --- a/src/lib/util/unzip.cpp +++ b/src/lib/util/unzip.cpp @@ -11,9 +11,9 @@ #include "unzip.h" #include "corestr.h" +#include "hashing.h" #include "osdcore.h" - #include <algorithm> #include <array> #include <cassert> @@ -25,63 +25,12 @@ #include <zlib.h> +#include "lzma/C/LzmaDec.h" + namespace util { namespace { /*************************************************************************** - CONSTANTS -***************************************************************************/ - -/* offsets in end of central directory structure */ -#define ZIPESIG 0x00 -#define ZIPEDSK 0x04 -#define ZIPECEN 0x06 -#define ZIPENUM 0x08 -#define ZIPECENN 0x0a -#define ZIPECSZ 0x0c -#define ZIPEOFST 0x10 -#define ZIPECOML 0x14 -#define ZIPECOM 0x16 - -/* offsets in central directory entry structure */ -#define ZIPCENSIG 0x00 -#define ZIPCVER 0x04 -#define ZIPCOS 0x05 -#define ZIPCVXT 0x06 -#define ZIPCEXOS 0x07 -#define ZIPCFLG 0x08 -#define ZIPCMTHD 0x0a -#define ZIPCTIM 0x0c -#define ZIPCDAT 0x0e -#define ZIPCCRC 0x10 -#define ZIPCSIZ 0x14 -#define ZIPCUNC 0x18 -#define ZIPCFNL 0x1c -#define ZIPCXTL 0x1e -#define ZIPCCML 0x20 -#define ZIPDSK 0x22 -#define ZIPINT 0x24 -#define ZIPEXT 0x26 -#define ZIPOFST 0x2a -#define ZIPCFN 0x2e - -/* offsets in local file header structure */ -#define ZIPLOCSIG 0x00 -#define ZIPVER 0x04 -#define ZIPGENFLG 0x06 -#define ZIPMTHD 0x08 -#define ZIPTIME 0x0a -#define ZIPDATE 0x0c -#define ZIPCRC 0x0e -#define ZIPSIZE 0x12 -#define ZIPUNCMP 0x16 -#define ZIPFNLN 0x1a -#define ZIPXTRALN 0x1c -#define ZIPNAME 0x1e - - - -/*************************************************************************** TYPE DEFINITIONS ***************************************************************************/ @@ -99,10 +48,8 @@ public: , m_cd_pos(0) , m_header() , m_curr_is_dir(false) - , m_curr_name() , m_buffer() { - std::memset(&m_header, 0, sizeof(m_header)); std::fill(m_buffer.begin(), m_buffer.end(), 0); } @@ -137,18 +84,23 @@ public: return ziperr; // verify that we can work with this zipfile (no disk spanning allowed) - if ((m_ecd.disk_number != m_ecd.cd_start_disk_number) || (m_ecd.cd_disk_entries != m_ecd.cd_total_entries)) + if ((m_ecd.disk_number != m_ecd.cd_start_disk_number) || + (m_ecd.cd_disk_entries != m_ecd.cd_total_entries) || + (std::size_t(m_ecd.cd_size) != m_ecd.cd_size)) return archive_file::error::UNSUPPORTED; // allocate memory for the central directory - try { m_cd.resize(m_ecd.cd_size + 1); } + try { m_cd.resize(std::size_t(m_ecd.cd_size)); } catch (...) { return archive_file::error::OUT_OF_MEMORY; } // read the central directory + // FIXME: central directory can theoretically be bigger than maximum 32-bit integer value, should use a loop std::uint32_t read_length; auto const filerr = m_file->read(&m_cd[0], m_ecd.cd_start_disk_offset, m_ecd.cd_size, read_length); - if ((filerr != osd_file::error::NONE) || (read_length != m_ecd.cd_size)) - return (filerr == osd_file::error::NONE) ? archive_file::error::FILE_TRUNCATED : archive_file::error::FILE_ERROR; + if (filerr != osd_file::error::NONE) + return archive_file::error::FILE_ERROR; + if (read_length != m_ecd.cd_size) + return archive_file::error::FILE_TRUNCATED; return archive_file::error::NONE; } @@ -180,7 +132,7 @@ public: } bool current_is_directory() const { return m_curr_is_dir; } - const std::string ¤t_name() const { return m_curr_name; } + const std::string ¤t_name() const { return m_header.file_name; } std::uint64_t current_uncompressed_length() const { return m_header.uncompressed_length; } std::uint32_t current_crc() const { return m_header.crc; } @@ -212,44 +164,31 @@ private: // decompression interfaces archive_file::error decompress_data_type_0(std::uint64_t offset, void *buffer, std::uint32_t length); archive_file::error decompress_data_type_8(std::uint64_t offset, void *buffer, std::uint32_t length); + archive_file::error decompress_data_type_14(std::uint64_t offset, void *buffer, std::uint32_t length); struct file_header { - std::uint32_t signature; // central file header signature std::uint16_t version_created; // version made by std::uint16_t version_needed; // version needed to extract std::uint16_t bit_flag; // general purpose bit flag std::uint16_t compression; // compression method - std::uint16_t file_time; // last mod file time - std::uint16_t file_date; // last mod file date std::uint32_t crc; // crc-32 - std::uint32_t compressed_length; // compressed size - std::uint32_t uncompressed_length; // uncompressed size - std::uint16_t filename_length; // filename length - std::uint16_t extra_field_length; // extra field length - std::uint16_t file_comment_length; // file comment length - std::uint16_t start_disk_number; // disk number start - std::uint16_t internal_attributes; // internal file attributes - std::uint32_t external_attributes; // external file attributes - std::uint32_t local_header_offset; // relative offset of local header - const char * filename; // filename + std::uint64_t compressed_length; // compressed size + std::uint64_t uncompressed_length; // uncompressed size + std::uint32_t start_disk_number; // disk number start + std::uint64_t local_header_offset; // relative offset of local header + std::string file_name; // file name }; // contains extracted end of central directory information struct ecd { - std::uint32_t signature; // end of central dir signature - std::uint16_t disk_number; // number of this disk - std::uint16_t cd_start_disk_number; // number of the disk with the start of the central directory - std::uint16_t cd_disk_entries; // total number of entries in the central directory on this disk - std::uint16_t cd_total_entries; // total number of entries in the central directory - std::uint32_t cd_size; // size of the central directory - std::uint32_t cd_start_disk_offset; // offset of start of central directory with respect to the starting disk number - std::uint16_t comment_length; // .ZIP file comment length - const char * comment; // .ZIP file comment - - std::unique_ptr<std::uint8_t []> raw; // pointer to the raw data - std::uint32_t rawlength; // length of the raw data + std::uint32_t disk_number; // number of this disk + std::uint32_t cd_start_disk_number; // number of the disk with the start of the central directory + std::uint64_t cd_disk_entries; // total number of entries in the central directory on this disk + std::uint64_t cd_total_entries; // total number of entries in the central directory + std::uint64_t cd_size; // size of the central directory + std::uint64_t cd_start_disk_offset; // offset of start of central directory with respect to the starting disk number }; static constexpr std::size_t DECOMPRESS_BUFSIZE = 16384; @@ -267,7 +206,6 @@ private: std::uint32_t m_cd_pos; // position in central directory file_header m_header; // current file header bool m_curr_is_dir; // current file is directory - std::string m_curr_name; // current file name std::array<std::uint8_t, DECOMPRESS_BUFSIZE> m_buffer; // buffer for decompression }; @@ -307,40 +245,282 @@ private: }; +class reader_base +{ +protected: + reader_base(void const *buf) : m_buffer(reinterpret_cast<std::uint8_t const *>(buf)) { } + + std::uint8_t read_byte(std::size_t offs) const + { + return m_buffer[offs]; + } + std::uint16_t read_word(std::size_t offs) const + { + return + (std::uint16_t(m_buffer[offs + 1]) << 8) | + (std::uint16_t(m_buffer[offs + 0]) << 0); + } + std::uint32_t read_dword(std::size_t offs) const + { + return + (std::uint32_t(m_buffer[offs + 3]) << 24) | + (std::uint32_t(m_buffer[offs + 2]) << 16) | + (std::uint32_t(m_buffer[offs + 1]) << 8) | + (std::uint32_t(m_buffer[offs + 0]) << 0); + } + std::uint64_t read_qword(std::size_t offs) const + { + return + (std::uint64_t(m_buffer[offs + 7]) << 56) | + (std::uint64_t(m_buffer[offs + 6]) << 48) | + (std::uint64_t(m_buffer[offs + 5]) << 40) | + (std::uint64_t(m_buffer[offs + 4]) << 32) | + (std::uint64_t(m_buffer[offs + 3]) << 24) | + (std::uint64_t(m_buffer[offs + 2]) << 16) | + (std::uint64_t(m_buffer[offs + 1]) << 8) | + (std::uint64_t(m_buffer[offs + 0]) << 0); + } + std::string read_string(std::size_t offs, std::string::size_type len) const + { + return std::string(reinterpret_cast<char const *>(m_buffer + offs), len); + } + void read_string(std::string &result, std::size_t offs, std::string::size_type len) const + { + result.assign(reinterpret_cast<char const *>(m_buffer + offs), len); + } + + std::uint8_t const *m_buffer; +}; -/*************************************************************************** - INLINE FUNCTIONS -***************************************************************************/ -/** - * @fn static inline UINT16 read_word(UINT8 *buf) - * - * @brief Reads a word. - * - * @param [in,out] buf If non-null, the buffer. - * - * @return The word. - */ - -inline std::uint16_t read_word(std::uint8_t const *buf) +class extra_field_reader : private reader_base { - return (buf[1] << 8) | buf[0]; -} +public: + extra_field_reader(void const *buf, std::size_t len) : reader_base(buf), m_length(len) { } + + std::uint16_t header_id() const { return read_word(0x00); } + std::uint16_t data_size() const { return read_word(0x02); } + void const * data() const { return m_buffer + 0x04; } + extra_field_reader next() const { return extra_field_reader(m_buffer + total_length(), m_length - total_length()); } -/** - * @fn static inline UINT32 read_dword(UINT8 *buf) - * - * @brief Reads a double word. - * - * @param [in,out] buf If non-null, the buffer. - * - * @return The double word. - */ - -inline std::uint32_t read_dword(std::uint8_t const *buf) + bool length_sufficient() const { return (m_length >= minimum_length()) && (m_length >= total_length()); } + + std::size_t total_length() const { return minimum_length() + data_size(); } + static std::size_t minimum_length() { return 0x04; } + +private: + std::size_t m_length; +}; + + +class local_file_header_reader : private reader_base { - return (buf[3] << 24) | (buf[2] << 16) | (buf[1] << 8) | buf[0]; -} +public: + local_file_header_reader(void const *buf) : reader_base(buf) { } + + std::uint32_t signature() const { return read_dword(0x00); } + std::uint8_t version_needed() const { return m_buffer[0x04]; } + std::uint8_t os_needed() const { return m_buffer[0x05]; } + std::uint16_t general_flag() const { return read_word(0x06); } + std::uint16_t compression_method() const { return read_word(0x08); } + std::uint16_t modified_time() const { return read_word(0x0a); } + std::uint16_t modified_date() const { return read_word(0x0c); } + std::uint32_t crc32() const { return read_dword(0x0e); } + std::uint32_t compressed_size() const { return read_dword(0x12); } + std::uint32_t uncompressed_size() const { return read_dword(0x16); } + std::uint16_t file_name_length() const { return read_word(0x1a); } + std::uint16_t extra_field_length() const { return read_word(0x1c); } + std::string file_name() const { return read_string(0x1e, file_name_length()); } + void file_name(std::string &result) const { read_string(result, 0x1e, file_name_length()); } + extra_field_reader extra_field() const { return extra_field_reader(m_buffer + 0x1e + file_name_length(), extra_field_length()); } + + bool signature_correct() const { return signature() == 0x04034b50; } + + std::size_t total_length() const { return minimum_length() + file_name_length() + extra_field_length(); } + static std::size_t minimum_length() { return 0x1e; } +}; + + +class central_dir_entry_reader : private reader_base +{ +public: + central_dir_entry_reader(void const *buf) : reader_base(buf) { } + + std::uint32_t signature() const { return read_dword(0x00); } + std::uint8_t version_created() const { return m_buffer[0x04]; } + std::uint8_t os_created() const { return m_buffer[0x05]; } + std::uint8_t version_needed() const { return m_buffer[0x06]; } + std::uint8_t os_needed() const { return m_buffer[0x07]; } + std::uint16_t general_flag() const { return read_word(0x08); } + std::uint16_t compression_method() const { return read_word(0x0a); } + std::uint16_t modified_time() const { return read_word(0x0c); } + std::uint16_t modified_date() const { return read_word(0x0e); } + std::uint32_t crc32() const { return read_dword(0x10); } + std::uint32_t compressed_size() const { return read_dword(0x14); } + std::uint32_t uncompressed_size() const { return read_dword(0x18); } + std::uint16_t file_name_length() const { return read_word(0x1c); } + std::uint16_t extra_field_length() const { return read_word(0x1e); } + std::uint16_t file_comment_length() const { return read_word(0x20); } + std::uint16_t start_disk() const { return read_word(0x22); } + std::uint16_t int_file_attr() const { return read_word(0x24); } + std::uint32_t ext_file_attr() const { return read_dword(0x26); } + std::uint32_t header_offset() const { return read_dword(0x2a); } + std::string file_name() const { return read_string(0x2e, file_name_length()); } + void file_name(std::string &result) const { read_string(result, 0x2e, file_name_length()); } + extra_field_reader extra_field() const { return extra_field_reader(m_buffer + 0x2e + file_name_length(), extra_field_length()); } + std::string file_comment() const { return read_string(0x2e + file_name_length() + extra_field_length(), file_comment_length()); } + void file_comment(std::string &result) const { read_string(result, 0x2e + file_name_length() + extra_field_length(), file_comment_length()); } + + bool signature_correct() const { return signature() == 0x02014b50; } + + std::size_t total_length() const { return minimum_length() + file_name_length() + extra_field_length() + file_comment_length(); } + static std::size_t minimum_length() { return 0x2e; } +}; + + +class ecd64_reader : private reader_base +{ +public: + ecd64_reader(void const *buf) : reader_base(buf) { } + + std::uint32_t signature() const { return read_dword(0x00); } + std::uint64_t ecd64_size() const { return read_qword(0x04); } + std::uint8_t version_created() const { return m_buffer[0x0c]; } + std::uint8_t os_created() const { return m_buffer[0x0d]; } + std::uint8_t version_needed() const { return m_buffer[0x0e]; } + std::uint8_t os_needed() const { return m_buffer[0x0f]; } + std::uint32_t this_disk_no() const { return read_dword(0x10); } + std::uint32_t dir_start_disk() const { return read_dword(0x14); } + std::uint64_t dir_disk_entries() const { return read_qword(0x18); } + std::uint64_t dir_total_entries() const { return read_qword(0x20); } + std::uint64_t dir_size() const { return read_qword(0x28); } + std::uint64_t dir_offset() const { return read_qword(0x30); } + void const * extensible_data() const { return m_buffer + 0x38; } + + bool signature_correct() const { return signature() == 0x06064b50; } + + std::size_t total_length() const { return 0x0c + ecd64_size(); } + static std::size_t minimum_length() { return 0x38; } +}; + + +class ecd64_locator_reader : private reader_base +{ +public: + ecd64_locator_reader(void const *buf) : reader_base(buf) { } + + std::uint32_t signature() const { return read_dword(0x00); } + std::uint32_t ecd64_disk() const { return read_dword(0x04); } + std::uint64_t ecd64_offset() const { return read_qword(0x08); } + std::uint32_t total_disks() const { return read_dword(0x10); } + + bool signature_correct() const { return signature() == 0x07064b50; } + + std::size_t total_length() const { return minimum_length(); } + static std::size_t minimum_length() { return 0x14; } +}; + + +class ecd_reader : private reader_base +{ +public: + ecd_reader(void const *buf) : reader_base(buf) { } + + std::uint32_t signature() const { return read_dword(0x00); } + std::uint16_t this_disk_no() const { return read_word(0x04); } + std::uint16_t dir_start_disk() const { return read_word(0x06); } + std::uint16_t dir_disk_entries() const { return read_word(0x08); } + std::uint16_t dir_total_entries() const { return read_word(0x0a); } + std::uint32_t dir_size() const { return read_dword(0x0c); } + std::uint32_t dir_offset() const { return read_dword(0x10); } + std::uint16_t comment_length() const { return read_word(0x14); } + std::string comment() const { return read_string(0x16, comment_length()); } + void comment(std::string &result) const { read_string(result, 0x16, comment_length()); } + + bool signature_correct() const { return signature() == 0x06054b50; } + + std::size_t total_length() const { return minimum_length() + comment_length(); } + static std::size_t minimum_length() { return 0x16; } +}; + + +class zip64_ext_info_reader : private reader_base +{ +public: + template <typename T> + zip64_ext_info_reader( + T const &header, + extra_field_reader const &field) + : reader_base(field.data()) + , m_uncompressed_size(header.uncompressed_size()) + , m_compressed_size(header.compressed_size()) + , m_header_offset(header.header_offset()) + , m_start_disk(header.start_disk()) + , m_offs_compressed_size(~m_uncompressed_size ? 0 : 8) + , m_offs_header_offset(m_offs_compressed_size + (~m_compressed_size ? 0 : 8)) + , m_offs_start_disk(m_offs_header_offset + (~m_header_offset ? 0 : 8)) + , m_offs_end(m_offs_start_disk + (~m_start_disk ? 0 : 4)) + { + } + + std::uint64_t uncompressed_size() const { return ~m_uncompressed_size ? m_uncompressed_size : read_qword(0x00); } + std::uint64_t compressed_size() const { return ~m_compressed_size ? m_compressed_size : read_qword(m_offs_compressed_size); } + std::uint64_t header_offset() const { return ~m_header_offset ? m_header_offset : read_qword(m_offs_header_offset); } + std::uint32_t start_disk() const { return ~m_start_disk ? m_start_disk : read_dword(m_offs_start_disk); } + + std::size_t total_length() const { return minimum_length() + m_offs_end; } + static std::size_t minimum_length() { return 0x00; } + +private: + std::uint32_t m_uncompressed_size; + std::uint32_t m_compressed_size; + std::uint32_t m_header_offset; + std::uint16_t m_start_disk; + + std::size_t m_offs_compressed_size; + std::size_t m_offs_header_offset; + std::size_t m_offs_start_disk; + std::size_t m_offs_end; +}; + + +class utf8_path_reader : private reader_base +{ +public: + utf8_path_reader(extra_field_reader const &field) : reader_base(field.data()), m_length(field.data_size()) { } + + std::uint8_t version() const { return m_buffer[0]; } + std::uint32_t name_crc32() const { return read_dword(0x01); } + std::string unicode_name() const { return read_string(0x05, m_length - 0x05); } + void unicode_name(std::string &result) const { return read_string(result, 0x05, m_length - 0x05); } + + std::size_t total_length() const { return m_length; } + static std::size_t minimum_length() { return 0x05; } + +private: + std::size_t m_length; +}; + + +class general_flag_reader +{ +public: + general_flag_reader(std::uint16_t val) : m_value(val) { } + + bool encrypted() const { return bool(m_value & 0x0001); } + bool implode_8k_dict() const { return bool(m_value & 0x0002); } + bool implode_3_trees() const { return bool(m_value & 0x0004); } + unsigned deflate_option() const { return unsigned((m_value >> 1) & 0x0003); } + bool lzma_eos_mark() const { return bool(m_value & 0x0002); } + bool use_descriptor() const { return bool(m_value & 0x0008); } + bool patch_data() const { return bool(m_value & 0x0020); } + bool strong_encryption() const { return bool(m_value & 0x0040); } + bool utf8_encoding() const { return bool(m_value & 0x0800); } + bool directory_encryption() const { return bool(m_value & 0x2000); } + +private: + std::uint16_t m_value; +}; @@ -359,14 +539,6 @@ std::mutex zip_file_impl::s_cache_mutex; to the cache -------------------------------------------------*/ -/** - * @fn void zip_file_close(zip_file *zip) - * - * @brief Zip file close. - * - * @param [in,out] zip If non-null, the zip. - */ - void zip_file_impl::close(ptr &&zip) { if (!zip) return; @@ -397,67 +569,90 @@ void zip_file_impl::close(ptr &&zip) ***************************************************************************/ /*------------------------------------------------- - zip_file_first_entry - return the first entry - in the ZIP --------------------------------------------------*/ - -/*------------------------------------------------- - zip_file_next_entry - return the next entry - in the ZIP + zip_file_search - return the next matching + entry in the ZIP -------------------------------------------------*/ int zip_file_impl::search(std::uint32_t search_crc, const std::string &search_filename, bool matchcrc, bool matchname, bool partialpath) { // if we're at or past the end, we're done - std::string filename; - while ((m_cd_pos + ZIPCFN) <= m_ecd.cd_size) + while ((m_cd_pos + central_dir_entry_reader::minimum_length()) <= m_ecd.cd_size) { - // extract file header info - std::uint8_t const *const raw = &m_cd[0] + m_cd_pos; - m_header.signature = read_dword(raw + ZIPCENSIG); - m_header.version_created = read_word (raw + ZIPCVER); - m_header.version_needed = read_word (raw + ZIPCVXT); - m_header.bit_flag = read_word (raw + ZIPCFLG); - m_header.compression = read_word (raw + ZIPCMTHD); - m_header.file_time = read_word (raw + ZIPCTIM); - m_header.file_date = read_word (raw + ZIPCDAT); - m_header.crc = read_dword(raw + ZIPCCRC); - m_header.compressed_length = read_dword(raw + ZIPCSIZ); - m_header.uncompressed_length = read_dword(raw + ZIPCUNC); - m_header.filename_length = read_word (raw + ZIPCFNL); - m_header.extra_field_length = read_word (raw + ZIPCXTL); - m_header.file_comment_length = read_word (raw + ZIPCCML); - m_header.start_disk_number = read_word (raw + ZIPDSK); - m_header.internal_attributes = read_word (raw + ZIPINT); - m_header.external_attributes = read_dword(raw + ZIPEXT); - m_header.local_header_offset = read_dword(raw + ZIPOFST); - m_header.filename = reinterpret_cast<const char *>(raw + ZIPCFN); - // make sure we have enough data - std::uint32_t const rawlength = ZIPCFN + m_header.filename_length + m_header.extra_field_length + m_header.file_comment_length; - if ((m_cd_pos + rawlength) > m_ecd.cd_size) + central_dir_entry_reader const reader(&m_cd[0] + m_cd_pos); + if (!reader.signature_correct() || ((m_cd_pos + reader.total_length()) > m_ecd.cd_size)) break; + // extract file header info + m_header.version_created = reader.version_created(); + m_header.version_needed = reader.version_needed(); + m_header.bit_flag = reader.general_flag(); + m_header.compression = reader.compression_method(); + m_header.crc = reader.crc32(); + m_header.compressed_length = reader.compressed_size(); + m_header.uncompressed_length = reader.uncompressed_size(); + m_header.start_disk_number = reader.start_disk(); + m_header.local_header_offset = reader.header_offset(); + // advance the position - m_cd_pos += rawlength; + m_cd_pos += reader.total_length(); + + // copy the filename + bool is_utf8(general_flag_reader(m_header.bit_flag).utf8_encoding()); + reader.file_name(m_header.file_name); - // copy the filename filename - bool const is_dir((m_header.filename_length > 0) && (m_header.filename[m_header.filename_length - 1] == '/')); - filename.assign(m_header.filename, m_header.filename_length - (is_dir ? 1 : 0)); + // walk the extra data + for (auto extra = reader.extra_field(); extra.length_sufficient(); extra = extra.next()) + { + // look for ZIP64 extended info + if ((extra.header_id() == 0x0001) && (extra.data_size() >= zip64_ext_info_reader::minimum_length())) + { + zip64_ext_info_reader const ext64(reader, extra); + if (extra.data_size() >= ext64.total_length()) + { + m_header.compressed_length = ext64.compressed_size(); + m_header.uncompressed_length = ext64.uncompressed_size(); + m_header.start_disk_number = ext64.start_disk(); + m_header.local_header_offset = ext64.header_offset(); + } + } + + // look for Info-ZIP UTF-8 path + if (!is_utf8 && (extra.header_id() == 0x7075) && (extra.data_size() >= utf8_path_reader::minimum_length())) + { + utf8_path_reader const utf8path(extra); + if (utf8path.version() == 1) + { + auto const addr(m_header.file_name.empty() ? nullptr : &m_header.file_name[0]); + auto const length(m_header.file_name.empty() ? 0 : m_header.file_name.length() * sizeof(m_header.file_name[0])); + auto const crc(crc32_creator::simple(addr, length)); + if (utf8path.name_crc32() == crc.m_raw) + { + utf8path.unicode_name(m_header.file_name); + is_utf8 = true; + } + } + } + } + + // FIXME: if (!is_utf8) convert filename to UTF8 (assume CP437 or something) + + // chop off trailing slash for directory entries + bool const is_dir(!m_header.file_name.empty() && (m_header.file_name.back() == '/')); + if (is_dir) m_header.file_name.resize(m_header.file_name.length() - 1); // check to see if it matches query bool const crcmatch(search_crc == m_header.crc); - auto const partialoffset(filename.length() - search_filename.length()); - bool const partialpossible((filename.length() > search_filename.length()) && (filename[partialoffset - 1] == '/')); + auto const partialoffset(m_header.file_name.length() - search_filename.length()); + bool const partialpossible((m_header.file_name.length() > search_filename.length()) && (m_header.file_name[partialoffset - 1] == '/')); const bool namematch( - !core_stricmp(search_filename.c_str(), filename.c_str()) || - (partialpath && partialpossible && !core_stricmp(search_filename.c_str(), filename.c_str() + partialoffset))); + !core_stricmp(search_filename.c_str(), m_header.file_name.c_str()) || + (partialpath && partialpossible && !core_stricmp(search_filename.c_str(), m_header.file_name.c_str() + partialoffset))); bool const found = ((!matchcrc && !matchname) || !is_dir) && (!matchcrc || crcmatch) && (!matchname || namematch); if (found) { m_curr_is_dir = is_dir; - m_curr_name = std::move(filename); return 0; } } @@ -470,18 +665,6 @@ int zip_file_impl::search(std::uint32_t search_crc, const std::string &search_fi from a ZIP into the target buffer -------------------------------------------------*/ -/** - * @fn zip_error zip_file_decompress(zip_file *zip, void *buffer, UINT32 length) - * - * @brief Zip file decompress. - * - * @param [in,out] zip If non-null, the zip. - * @param [in,out] buffer If non-null, the buffer. - * @param length The length. - * - * @return A zip_error. - */ - archive_file::error zip_file_impl::decompress(void *buffer, std::uint32_t length) { archive_file::error ziperr; @@ -511,6 +694,10 @@ archive_file::error zip_file_impl::decompress(void *buffer, std::uint32_t length ziperr = decompress_data_type_8(offset, buffer, length); break; + case 14: + ziperr = decompress_data_type_14(offset, buffer, length); + break; + default: ziperr = archive_file::error::UNSUPPORTED; break; @@ -528,16 +715,6 @@ archive_file::error zip_file_impl::decompress(void *buffer, std::uint32_t length read_ecd - read the ECD data -------------------------------------------------*/ -/** - * @fn static zip_error read_ecd(zip_file *zip) - * - * @brief Reads an ecd. - * - * @param [in,out] zip If non-null, the zip. - * - * @return The ecd. - */ - archive_file::error zip_file_impl::read_ecd() { // make sure the file handle is open @@ -546,6 +723,8 @@ archive_file::error zip_file_impl::read_ecd() return ziperr; // we may need multiple tries + osd_file::error error; + std::uint32_t read_length; std::uint32_t buflen = 1024; while (buflen < 65536) { @@ -559,38 +738,77 @@ archive_file::error zip_file_impl::read_ecd() catch (...) { return archive_file::error::OUT_OF_MEMORY; } // read in one buffers' worth of data - std::uint32_t read_length; - auto const error = m_file->read(&buffer[0], m_length - buflen, buflen, read_length); - if (error != osd_file::error::NONE || read_length != buflen) + error = m_file->read(&buffer[0], m_length - buflen, buflen, read_length); + if ((error != osd_file::error::NONE) || (read_length != buflen)) return archive_file::error::FILE_ERROR; // find the ECD signature std::int32_t offset; - for (offset = buflen - 22; offset >= 0; offset--) - if (buffer[offset + 0] == 'P' && buffer[offset + 1] == 'K' && buffer[offset + 2] == 0x05 && buffer[offset + 3] == 0x06) + for (offset = buflen - ecd_reader::minimum_length(); offset >= 0; offset--) + { + ecd_reader reader(buffer.get() + offset); + if (reader.signature_correct() && ((reader.total_length() + offset) <= buflen)) break; + } // if we found it, fill out the data if (offset >= 0) { - // reuse the buffer as our ECD buffer - m_ecd.raw = std::move(buffer); - m_ecd.rawlength = buflen - offset; - - /* append a NULL terminator to the comment */ - memmove(&m_ecd.raw[0], &m_ecd.raw[offset], m_ecd.rawlength); - m_ecd.raw[m_ecd.rawlength] = 0; - - /* extract ecd info */ - m_ecd.signature = read_dword(&m_ecd.raw[ZIPESIG]); - m_ecd.disk_number = read_word (&m_ecd.raw[ZIPEDSK]); - m_ecd.cd_start_disk_number = read_word (&m_ecd.raw[ZIPECEN]); - m_ecd.cd_disk_entries = read_word (&m_ecd.raw[ZIPENUM]); - m_ecd.cd_total_entries = read_word (&m_ecd.raw[ZIPECENN]); - m_ecd.cd_size = read_dword(&m_ecd.raw[ZIPECSZ]); - m_ecd.cd_start_disk_offset = read_dword(&m_ecd.raw[ZIPEOFST]); - m_ecd.comment_length = read_word (&m_ecd.raw[ZIPECOML]); - m_ecd.comment = reinterpret_cast<const char *>(&m_ecd.raw[ZIPECOM]); + // extract ECD info + ecd_reader const ecd_rd(buffer.get() + offset); + m_ecd.disk_number = ecd_rd.this_disk_no(); + m_ecd.cd_start_disk_number = ecd_rd.dir_start_disk(); + m_ecd.cd_disk_entries = ecd_rd.dir_disk_entries(); + m_ecd.cd_total_entries = ecd_rd.dir_total_entries(); + m_ecd.cd_size = ecd_rd.dir_size(); + m_ecd.cd_start_disk_offset = ecd_rd.dir_offset(); + + // is the file too small to contain a ZIP64 ECD locator? + if ((m_length - buflen + offset) < ecd64_locator_reader::minimum_length()) + return archive_file::error::NONE; + + // try to read the ZIP64 ECD locator + error = m_file->read( + &buffer[0], + m_length - buflen + offset - ecd64_locator_reader::minimum_length(), + ecd64_locator_reader::minimum_length(), + read_length); + if ((error != osd_file::error::NONE) || (read_length != ecd64_locator_reader::minimum_length())) + return archive_file::error::FILE_ERROR; + + // if the signature isn't correct, it's not a ZIP64 archive + ecd64_locator_reader const ecd64_loc_rd(buffer.get()); + if (!ecd64_loc_rd.signature_correct()) + return archive_file::error::NONE; + + // check that the ZIP64 ECD is in this segment (assuming this segment is the last segment) + if ((ecd64_loc_rd.ecd64_disk() + 1) != ecd64_loc_rd.total_disks()) + return archive_file::error::UNSUPPORTED; + + // try to read the ZIP64 ECD + error = m_file->read(&buffer[0], ecd64_loc_rd.ecd64_offset(), ecd64_reader::minimum_length(), read_length); + if (error != osd_file::error::NONE) + return archive_file::error::FILE_ERROR; + if (read_length != ecd64_reader::minimum_length()) + return archive_file::error::FILE_TRUNCATED; + + // check ZIP64 ECD + ecd64_reader const ecd64_rd(buffer.get()); + if (!ecd64_rd.signature_correct()) + return archive_file::error::BAD_SIGNATURE; + if (ecd64_rd.total_length() < ecd64_reader::minimum_length()) + return archive_file::error::UNSUPPORTED; + if (ecd64_rd.version_needed() > 63) + return archive_file::error::UNSUPPORTED; + + // extract ZIP64 ECD info + m_ecd.disk_number = ecd64_rd.this_disk_no(); + m_ecd.cd_start_disk_number = ecd64_rd.dir_start_disk(); + m_ecd.cd_disk_entries = ecd64_rd.dir_disk_entries(); + m_ecd.cd_total_entries = ecd64_rd.dir_total_entries(); + m_ecd.cd_size = ecd64_rd.dir_size(); + m_ecd.cd_start_disk_offset = ecd64_rd.dir_offset(); + return archive_file::error::NONE; } @@ -609,19 +827,17 @@ archive_file::error zip_file_impl::read_ecd() offset of the compressed data -------------------------------------------------*/ -/** - * @fn static zip_error get_compressed_data_offset(zip_file *zip, UINT64 *offset) - * - * @brief Gets compressed data offset. - * - * @param [in,out] zip If non-null, the zip. - * @param [in,out] offset If non-null, the offset. - * - * @return The compressed data offset. - */ - archive_file::error zip_file_impl::get_compressed_data_offset(std::uint64_t &offset) { + // don't support a number of features + general_flag_reader const flags(m_header.bit_flag); + if ((m_header.start_disk_number != m_ecd.disk_number) || + (m_header.version_needed > 63) || + flags.encrypted() || + flags.patch_data() || + flags.strong_encryption()) + return archive_file::error::UNSUPPORTED; + // make sure the file handle is open auto const ziperr = reopen(); if (ziperr != archive_file::error::NONE) @@ -629,14 +845,17 @@ archive_file::error zip_file_impl::get_compressed_data_offset(std::uint64_t &off // now go read the fixed-sized part of the local file header std::uint32_t read_length; - auto const error = m_file->read(&m_buffer[0], m_header.local_header_offset, ZIPNAME, read_length); - if (error != osd_file::error::NONE || read_length != ZIPNAME) - return (error == osd_file::error::NONE) ? archive_file::error::FILE_TRUNCATED : archive_file::error::FILE_ERROR; + auto const error = m_file->read(&m_buffer[0], m_header.local_header_offset, local_file_header_reader::minimum_length(), read_length); + if (error != osd_file::error::NONE) + return archive_file::error::FILE_ERROR; + if (read_length != local_file_header_reader::minimum_length()) + return archive_file::error::FILE_TRUNCATED; - /* compute the final offset */ - offset = m_header.local_header_offset + ZIPNAME; - offset += read_word(&m_buffer[ZIPFNLN]); - offset += read_word(&m_buffer[ZIPXTRALN]); + // compute the final offset + local_file_header_reader reader(&m_buffer[0]); + if (!reader.signature_correct()) + return archive_file::error::BAD_SIGNATURE; + offset = m_header.local_header_offset + reader.total_length(); return archive_file::error::NONE; } @@ -652,19 +871,6 @@ archive_file::error zip_file_impl::get_compressed_data_offset(std::uint64_t &off type 0 data (which is uncompressed) -------------------------------------------------*/ -/** - * @fn static zip_error decompress_data_type_0(zip_file *zip, UINT64 offset, void *buffer, UINT32 length) - * - * @brief Decompress the data type 0. - * - * @param [in,out] zip If non-null, the zip. - * @param offset The offset. - * @param [in,out] buffer If non-null, the buffer. - * @param length The length. - * - * @return A zip_error. - */ - archive_file::error zip_file_impl::decompress_data_type_0(std::uint64_t offset, void *buffer, std::uint32_t length) { std::uint32_t read_length; @@ -685,32 +891,18 @@ archive_file::error zip_file_impl::decompress_data_type_0(std::uint64_t offset, type 8 data (which is deflated) -------------------------------------------------*/ -/** - * @fn static zip_error decompress_data_type_8(zip_file *zip, UINT64 offset, void *buffer, UINT32 length) - * - * @brief Decompress the data type 8. - * - * @param [in,out] zip If non-null, the zip. - * @param offset The offset. - * @param [in,out] buffer If non-null, the buffer. - * @param length The length. - * - * @return A zip_error. - */ - archive_file::error zip_file_impl::decompress_data_type_8(std::uint64_t offset, void *buffer, std::uint32_t length) { - std::uint32_t input_remaining = m_header.compressed_length; + std::uint64_t input_remaining = m_header.compressed_length; int zerr; - // make sure we don't need a newer mechanism - if (m_header.version_needed > 0x14) - return archive_file::error::UNSUPPORTED; - - /* reset the stream */ + // reset the stream z_stream stream; - memset(&stream, 0, sizeof(stream)); - stream.next_out = (Bytef *)buffer; + stream.zalloc = Z_NULL; + stream.zfree = Z_NULL; + stream.opaque = Z_NULL; + stream.avail_in = 0; + stream.next_out = reinterpret_cast<Bytef *>(buffer); stream.avail_out = length; // initialize the decompressor @@ -723,7 +915,11 @@ archive_file::error zip_file_impl::decompress_data_type_8(std::uint64_t offset, { // read in the next chunk of data std::uint32_t read_length; - auto const filerr = m_file->read(&m_buffer[0], offset, (std::min<std::uint32_t>)(input_remaining, m_buffer.size()), read_length); + auto const filerr = m_file->read( + &m_buffer[0], + offset, + std::uint32_t((std::min<std::uint64_t>)(input_remaining, m_buffer.size())), + read_length); if (filerr != osd_file::error::NONE) { inflateEnd(&stream); @@ -732,7 +928,7 @@ archive_file::error zip_file_impl::decompress_data_type_8(std::uint64_t offset, offset += read_length; // if we read nothing, but still have data left, the file is truncated - if (read_length == 0 && input_remaining > 0) + if ((read_length == 0) && (input_remaining > 0)) { inflateEnd(&stream); return archive_file::error::FILE_TRUNCATED; @@ -763,13 +959,132 @@ archive_file::error zip_file_impl::decompress_data_type_8(std::uint64_t offset, if (zerr != Z_OK) return archive_file::error::DECOMPRESS_ERROR; - /* if anything looks funny, report an error */ - if (stream.avail_out > 0 || input_remaining > 0) + // if anything looks funny, report an error + if ((stream.avail_out > 0) || (input_remaining > 0)) return archive_file::error::DECOMPRESS_ERROR; return archive_file::error::NONE; } + +/*------------------------------------------------- + decompress_data_type_14 - decompress + type 14 data (LZMA) +-------------------------------------------------*/ + +archive_file::error zip_file_impl::decompress_data_type_14(std::uint64_t offset, void *buffer, std::uint32_t length) +{ + // two-byte version + // two-byte properties size (little-endian) + // properties + // compressed data + + assert(4 <= m_buffer.size()); + assert(LZMA_PROPS_SIZE <= m_buffer.size()); + + bool const eos_mark(general_flag_reader(m_header.bit_flag).lzma_eos_mark()); + Byte *const output(reinterpret_cast<Byte *>(buffer)); + SizeT output_remaining(length); + std::uint64_t input_remaining(m_header.compressed_length); + + std::uint32_t read_length; + osd_file::error filerr; + SRes lzerr; + ELzmaStatus lzstatus(LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK); + + // reset the stream + ISzAlloc alloc_imp; + alloc_imp.Alloc = [](void *p, std::size_t size) -> void * { return (size == 0) ? nullptr : std::malloc(size); }; + alloc_imp.Free = [](void *p, void *address) -> void { std::free(address); }; + CLzmaDec stream; + LzmaDec_Construct(&stream); + + // need to read LZMA properties before we can initialise the decompressor + if (4 > input_remaining) + return archive_file::error::DECOMPRESS_ERROR; + filerr = m_file->read(&m_buffer[0], offset, 4, read_length); + if (filerr != osd_file::error::NONE) + return archive_file::error::FILE_ERROR; + offset += read_length; + input_remaining -= read_length; + if (4 != read_length) + return archive_file::error::FILE_TRUNCATED; + std::uint16_t const props_size((std::uint16_t(m_buffer[3]) << 8) | std::uint16_t(m_buffer[2])); + if (props_size > m_buffer.size()) + return archive_file::error::UNSUPPORTED; + else if (props_size > input_remaining) + return archive_file::error::DECOMPRESS_ERROR; + filerr = m_file->read(&m_buffer[0], offset, props_size, read_length); + if (filerr != osd_file::error::NONE) + return archive_file::error::FILE_ERROR; + offset += read_length; + input_remaining -= read_length; + if (props_size != read_length) + return archive_file::error::FILE_TRUNCATED; + + // initialize the decompressor + lzerr = LzmaDec_Allocate(&stream, &m_buffer[0], props_size, &alloc_imp); + if (SZ_ERROR_MEM == lzerr) + return archive_file::error::OUT_OF_MEMORY; + else if (SZ_OK != lzerr) + return archive_file::error::DECOMPRESS_ERROR; + LzmaDec_Init(&stream); + + // loop until we're done + while (0 < input_remaining) + { + // read in the next chunk of data + filerr = m_file->read( + &m_buffer[0], + offset, + std::uint32_t((std::min<std::uint64_t>)(input_remaining, m_buffer.size())), + read_length); + if (filerr != osd_file::error::NONE) + { + LzmaDec_Free(&stream, &alloc_imp); + return archive_file::error::FILE_ERROR; + } + offset += read_length; + input_remaining -= read_length; + + // if we read nothing, but still have data left, the file is truncated + if ((0 == read_length) && (0 < input_remaining)) + { + LzmaDec_Free(&stream, &alloc_imp); + return archive_file::error::FILE_TRUNCATED; + } + + // now decompress + SizeT len(read_length); + // FIXME: is the input length really an in/out parameter or not? + // At least on reaching the end of the input, the input length doesn't seem to get updated + lzerr = LzmaDec_DecodeToBuf( + &stream, + output + length - output_remaining, + &output_remaining, + reinterpret_cast<Byte const *>(&m_buffer[0]), + &len, + ((0 == input_remaining) && eos_mark) ? LZMA_FINISH_END : LZMA_FINISH_ANY, + &lzstatus); + if (SZ_OK != lzerr) + { + LzmaDec_Free(&stream, &alloc_imp); + return archive_file::error::DECOMPRESS_ERROR; + } + } + + // finish decompression + LzmaDec_Free(&stream, &alloc_imp); + + // if anything looks funny, report an error + if (LZMA_STATUS_FINISHED_WITH_MARK == lzstatus) + return archive_file::error::NONE; + else if ((LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK != lzstatus) || eos_mark) + return archive_file::error::DECOMPRESS_ERROR; + else + return archive_file::error::NONE; +} + } // anonymous namespace @@ -790,17 +1105,6 @@ void m7z_file_cache_clear(); zip_file_open - opens a ZIP file for reading -------------------------------------------------*/ -/** - * @fn zip_error zip_file_open(const char *filename, zip_file **zip) - * - * @brief Queries if a given zip file open. - * - * @param filename Filename of the file. - * @param [in,out] zip If non-null, the zip. - * - * @return A zip_error. - */ - archive_file::error archive_file::open_zip(const std::string &filename, ptr &result) { // ensure we start with a NULL result diff --git a/src/lib/util/zippath.cpp b/src/lib/util/zippath.cpp index f16dcf824e6..1e4fb6dd293 100644 --- a/src/lib/util/zippath.cpp +++ b/src/lib/util/zippath.cpp @@ -17,6 +17,7 @@ #include <cassert> #include <cctype> +#include <forward_list> #include <new> @@ -26,20 +27,6 @@ namespace util { ***************************************************************************/ /** - * @struct zippath_returned_directory - * - * @brief A zippath returned directory. - */ - -struct zippath_returned_directory -{ - /** @brief The next. */ - zippath_returned_directory *next; - /** @brief The name. */ - std::string name; -}; - -/** * @class zippath_directory * * @brief A zippath directory. @@ -49,11 +36,13 @@ class zippath_directory { public: zippath_directory() - : returned_parent(false), - directory(nullptr), - called_zip_first(false), - zipfile(nullptr), - returned_dirlist(nullptr) { } + : returned_parent(false) + , directory(nullptr) + , called_zip_first(false) + , zipfile(nullptr) + , returned_dirlist() + { + } /* common */ /** @brief true to returned parent. */ @@ -73,7 +62,7 @@ public: /** @brief The zipprefix. */ std::string zipprefix; /** @brief The returned dirlist. */ - zippath_returned_directory *returned_dirlist; + std::forward_list<std::string> returned_dirlist; }; @@ -856,12 +845,7 @@ void zippath_closedir(zippath_directory *directory) if (directory->zipfile != nullptr) directory->zipfile.reset(); - while (directory->returned_dirlist != nullptr) - { - zippath_returned_directory *dirlist = directory->returned_dirlist; - directory->returned_dirlist = directory->returned_dirlist->next; - delete dirlist; - } + directory->returned_dirlist.clear(); delete directory; } @@ -887,24 +871,28 @@ void zippath_closedir(zippath_directory *directory) static const char *get_relative_path(zippath_directory const &directory) { auto len = directory.zipprefix.length(); - const char *prefix = directory.zipprefix.c_str(); + char const *prefix = directory.zipprefix.c_str(); while (is_zip_file_separator(*prefix)) { len--; prefix++; } - if ((len <= directory.zipfile->current_name().length()) && - !strncmp(prefix, directory.zipfile->current_name().c_str(), len)) + std::string const ¤t(directory.zipfile->current_name()); + char const *result = directory.zipfile->current_name().c_str() + len; + if ((current.length() >= len) && + !strncmp(prefix, current.c_str(), len) && + (!*prefix || is_zip_file_separator(*result) || is_zip_file_separator(directory.zipprefix.back()))) { - const char *result = &directory.zipfile->current_name().c_str()[len]; while (is_zip_file_separator(*result)) result++; return *result ? result : nullptr; } - - return nullptr; + else + { + return nullptr; + } } @@ -925,11 +913,6 @@ static const char *get_relative_path(zippath_directory const &directory) const osd_directory_entry *zippath_readdir(zippath_directory *directory) { const osd_directory_entry *result = nullptr; - int header; - const char *relpath; - const char *separator; - const char *s; - zippath_returned_directory *rdent; if (!directory->returned_parent) { @@ -947,10 +930,10 @@ const osd_directory_entry *zippath_readdir(zippath_directory *directory) { result = osd_readdir(directory->directory); } - while((result != nullptr) && (!strcmp(result->name, ".") || !strcmp(result->name, ".."))); + while (result && (!strcmp(result->name, ".") || !strcmp(result->name, ".."))); /* special case - is this entry a ZIP file? if so we need to return it as a "directory" */ - if ((result != nullptr) && (is_zip_file(result->name) || is_7z_file(result->name))) + if (result && (is_zip_file(result->name) || is_7z_file(result->name))) { /* copy; but change the entry type */ directory->returned_entry = *result; @@ -960,47 +943,46 @@ const osd_directory_entry *zippath_readdir(zippath_directory *directory) } else if (directory->zipfile) { + char const *relpath; do { /* a zip file read */ + int header; do { - if (!directory->called_zip_first) - header = directory->zipfile->first_file(); - else - header = directory->zipfile->next_file(); + header = directory->called_zip_first ? directory->zipfile->next_file() : directory->zipfile->first_file(); directory->called_zip_first = true; relpath = nullptr; } - while((header >= 0) && ((relpath = get_relative_path(*directory)) == nullptr)); + while ((header >= 0) && ((relpath = get_relative_path(*directory)) == nullptr)); - if (relpath != nullptr) + if (relpath) { /* we've found a ZIP entry; but this may be an entry deep within the target directory */ - for (s = relpath; *s && !is_zip_file_separator(*s); s++) - ; - separator = *s ? s : nullptr; + char const *separator = relpath; + while (*separator && !is_zip_file_separator(*separator)) separator++; - if (separator != nullptr) + if (*separator || directory->zipfile->current_is_directory()) { /* a nested entry; loop through returned_dirlist to see if we've returned the parent directory */ - for (rdent = directory->returned_dirlist; rdent != nullptr; rdent = rdent->next) + auto const len(separator - relpath); + auto rdent = directory->returned_dirlist.begin(); + while (directory->returned_dirlist.end() != rdent) { - if (!core_strnicmp(rdent->name.c_str(), relpath, separator - relpath)) + if ((rdent->length() == len) && !core_strnicmp(rdent->c_str(), relpath, len)) break; + else + ++rdent; } - if (rdent == nullptr) + if (directory->returned_dirlist.end() == rdent) { /* we've found a new directory; add this to returned_dirlist */ - rdent = new zippath_returned_directory; - rdent->next = directory->returned_dirlist; - rdent->name.assign(relpath, separator - relpath); - directory->returned_dirlist = rdent; + directory->returned_dirlist.emplace_front(relpath, separator - relpath); /* ...and return it */ memset(&directory->returned_entry, 0, sizeof(directory->returned_entry)); - directory->returned_entry.name = rdent->name.c_str(); + directory->returned_entry.name = directory->returned_dirlist.front().c_str(); directory->returned_entry.type = ENTTYPE_DIR; result = &directory->returned_entry; } @@ -1016,7 +998,7 @@ const osd_directory_entry *zippath_readdir(zippath_directory *directory) } } } - while((relpath != nullptr) && (result == nullptr)); + while (relpath && !result); } return result; } |