summaryrefslogtreecommitdiffstatshomepage
path: root/src/mame/drivers/lindbergh.c
blob: 2ddac448b4c4df79901e9e1609a3a19eda49496c (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
/***************************************************************************

    Sega Lindbergh skeleton driver

    TODO:
    - tests area 0xd0000 - 0xd000f, wants an undumped ROM in there?
    - Apparently there's no way to avoid a dead lock at 0xfd085, perhaps
      tied to the aforementioned?

***************************************************************************

Lindbergh
Sega 2005-2009

This is a "PC-based" arcade system. Different configurations have different colored boxes.
The version documented here is the red box. The PC part of it is mostly just the CPU,
Intel North/South-bridge chipset and AGP/PCI card slots etc. The main board is still
a typically custom-made Sega arcade PCB using a custom nVIDIA GeForce video card.
The main board also has a slot for a compact flash card. Primary storage media is DVD or HDD.
Both CF and HDD are locked and unreadable on a regular PC.

The familiar PIC is still present on the back of the system and likely decrypts the HDD and/or DVD.

On this red box the CPU is a Celeron D at 2.8GHz. RAM is 512M DDR PC3200
The box has Sega number 845-0001D-02


Mainboard
---------

838-14487
Sticker: 838-14673
                   |----|     |-----|
                   |USB ||---||1/8  | |-------|
                   |USB ||USB||AUDIO| |SERIAL1|  SECURITY
-------------------|RJ45||USB||JACKS|-|SERIAL2|--CONNECTOR---|
|OSC(D245L6I)                                                |
|      VIA                   OSC(D250L6I)                    |
|      VT1616          |-------|                             |
|                      |82541PI|  12V_POWER                  |
|                      |INTEL  |                             |
|  P  P  P  P          |-------|              |-----------|  |
|  C  C  C  C                        ISL6556B |           |  |
|  I  I  I  I         A          |-------|    |           |  |
|  4  3  2  1         G          |JG82875|    |           |  |
|                     P          |SL8DB  |    |    CPU    |  |
|                                |       |    |           |  |
|                                |INTEL  |    |           |  |
|          14.31818MHz           |-------|    |           |  |
|JP                                           |-----------|  |
|12                                                          |
|34  |-------| 932S208DG           SIMM1(512M DDR PC3200)    |
|56  |6300ESB|                                               |
|78  |SL7XJ  |                     SIMM2                     |
|910 |       |                                               |
|1112|INTEL  |                     SIMM3                     |
|    |-------|                                               |
|32.768kHz         |---------|     SIMM4                     |
| 3V_BATT          |COMPACT  |                               |
|    MB_BIOS.3J7   |FLASH    |     IDE40                     |
|                  |SLOT     |     IDE40       ATX_POWER     |
|------------------|---------|-------------------------------|
Notes:
           CPU - Intel Celeron D 335 SL8HM 2.8GHz 256k L2 cache, 533MHz FSB
       82541PI - Intel Gigabit Ethernet Controller
       6300ESB - Intel Southbridge IC
       JG82875 - Intel Northbridge IC
      ISL6556B - Intersil ISL6556B Optimized Multiphase PWM Controller with 6-Bit
                 DAC and Programmable Internal Temperature Compensation
      932S208DG- IDT 932S208DG Programmable PLL Clock synthesizer
        VT1616 - VIA VT1616 6-channel AC97 codec sound IC
            JP - Jumpers....
                 1-3 Normal (SET ON)
                 1-2 CMOS
                 3-4 PASSWORD
                 5-6 PCBL
                 7-8 BIOS
                 9-10 CF SLAVE
                 11-12 CF MASTER (SET ON)
         IDE40 - ATA133 IDE connector(s)
                 A 40GB HDD is plugged in via an 80-pin flat cable
                 This game is 'Too Spicy'. The hard drive is a Hitachi Deskstar
                 model HDS728040PLAT20. Capacity is 41GB. C/H/S 16383/16/63
                 LBA 80,418,240 sectors. In the model number 8040 means 80GB full
                 capacity but only 40GB is actually available
                 P/N: 0A30209 BA17730E6B
                 Serial: EETNGM0G
       CF SLOT - Accepts a compact flash card. The card is required to boot the system.
                 Revision C and E have been seen. There may be other revisions out there.
                 Sticker: LINDBERGH
                          MDA-C0004A
                          REV. C


Rear Board incorporating Security Board (plugged into main board security connector)
---------------------------------------

This board has the power input connectors on it and holes for access to the mainboard
I/O conections (LAN/COM/USB/speakers etc). Lower left is where the security board is plugged in.

837-14520R
171-8322C
839-1275
Sticker 839-1275R
|----------------------------------|
|                                  |
|                                  |
|       POWER       POWER          |
|                           CN5    |
|                                  |
| LED LED                          |
|                                  |
| DSW(8)                           |
|                                  |
|                                  |
|      SW2  SW1                    |
|                    C/W           |
|                                  |
|     CN2                          |
|-------------|COM1  SPK_REAR  LAN |
|             |                USB4|
|   DIP18.IC1 |                USB3|
|             |COM2  SPK_FR    USB2|
|SECURITY_CONN|                USB1|
|-------------|--------------------|
DSW(8)    - OFF,OFF,OFF,ON,ON,OFF,OFF,ON
DIP18.IC1 - DIP18 socket for protection PIC16F648A

                        Sega Security
            Game        ID Number      Dongle Sticker
            -----------------------------------------
            Too Spicy   317-0490-COM   253-5508-0491


Video Card (plugged into AGP slot)
----------

nVIDIA 180-10508-0000-A00
Sticker: BIOS VERSION 5.73.22.55.08
Sticker: 900-10508-2304-000 D 032 Made In China
Sticker: 600-10508-0004-000 J
Sticker: GeForce 7600 GS 0325206031558
 |----------------------------------------|
 |  VRAM *VRAM                VRAM *VRAM  |-|
 |                                        | |POWER_CONN
 | *VRAM  VRAM |---------|   *VRAM  VRAM  | |
|-V            |NVIDIA   |                |-|
| G            |U611B269 |                |
|-A            |0646B1S  |                |
 |       27MHz |NA6105.00P                |
 |             |G73-N-B1 |                |
 |             |---------|                |
 |                                        |
|-D  *RT9173C             |-------| 25MHz |
| V                       |NVIDIA |       |
| I      *VID_BIOS.U504   |HSI-N-A4       |
|-                        |4MJHT07B30 0607|
 |                        |-------|       |
 |    |----------|     AGP     |----------|
 |----|          |-------------|
Notes:
       * - These parts on the other side of the PCB
    VRAM - QIMONDA HYB18T256161AFL25 WVV10017 256Mbit x16 DDR2 SDRAM (P-TFBGA-84)
VID_BIOS - SST 25VF512 512Kbit serial flash ROM (video BIOS) at location U504 (SOIC8)


JVS I/O Card (plugged into PCI slot #4)
------------

Sega 2004
171-8300C
837-14472R
Sticker: 837-14472R91
|-----------------------------------------|
|                                  3V_BATT|
|        FLASH.IC6            DS14185     |
|                   D442012               |
|                                         |
|LED                                      |
|    DS485                                |
|USB                                0.1F  |
|                                   5.5v  |
|          EDS1232 |--------|       SUPERCAP
|   RTL8201        |FREESCALE             |
|                  |MPC8248 |             |
|RJ45   25MHz      |        |48MHz        |
|                  |        |58.9824MHz   |
| ISP1106          |--------|             |
|mUSB                            PQ070XZ1H|
|  |--------|      PCI      |-------------|
|--|        |---------------|
Notes:
      FLASH.IC6 - Spansion S29AL032D70 32Mbit flash ROM labelled 'FPR-24370B' (TSOP48)
      DS14185   - National Semiconductor DS14185 EIA/TIA-232 3 Driver x 5 Receiver (SOIC10)
      DS485     - National Semiconductor DS485 Low Power RS-485/RS-422 Multipoint Transceiver (SOIC8)
      MPC8248   - Freescale MPC8248 PowerQUICC II Family Multi-Channel Controller (PBGA516)
      EDS1232   - Elpida EDS1232AATA-75-E 128Mbit SDRAM (4M word x 32bit)
      D442012   - NEC D442012AGY-BB70 2Mbit CMOS Static RAM (128k-word x 16bit)
      ISP1106   - NXP Semiconductor ISP1106 Advanced Universal Serial Bus transceiver (SSOIC16)
      RTL8201   - Realtek RTL8201 Single Chip Single Pport 10/100M Fast Ethernet IC (QFP48)
      mUSB      - Mini USB connector


JVS I/O Board (connects arcade machine controls etc to PC)
-------------

Sega 2005
837-14572
171-8357B
|------------------------|
| USB_B USB_A CN6 CN7 CN8|
| GH6-2F                 |
| DS485   14.745MHz      |
|      UPC393            |
|           315-6414  CN9|
|                        |
|CN1 CN2       CN3       |
|------------------------|

*/

#include "emu.h"
#include "cpu/i386/i386.h"
#include "machine/pci.h"
#include "machine/i82875p.h"
#include "machine/i6300esb.h"
#include "machine/pci-usb.h"
#include "machine/pci-apic.h"
#include "machine/pci-sata.h"
#include "machine/pci-smbus.h"
#include "machine/i82541.h"
#include "machine/segabb.h"
#include "sound/pci-ac97.h"
#include "sound/sb0400.h"
#include "video/gf6800gt.h"

class lindbergh_state : public driver_device
{
public:
	lindbergh_state(const machine_config &mconfig, device_type type, const char *tag);

	virtual void machine_start();
	virtual void machine_reset();
};

#if 0
static ADDRESS_MAP_START(lindbergh_map, AS_PROGRAM, 32, lindbergh_state)
	AM_RANGE(0x00000000, 0x0009ffff) AM_RAM
									 //	AM_RANGE(0x000a0000, 0x000bffff) AM_DEVREADWRITE8("vga", vga_device, mem_r, mem_w, 0xffffffff)
									 //	AM_RANGE(0x000c0000, 0x000cffff) AM_ROM AM_REGION("vid_bios", 0)
//  0xd0000 - 0xdffff tested, wants 0x414d ("AM") in there
	AM_RANGE(0x000f0000, 0x000fffff) AM_ROM AM_REGION("mb_bios", 0xf0000)
//	AM_RANGE(0xfd000000, 0xfd3fffff) AM_ROM AM_REGION("jvs_bios", 0)    /* Hack to see the data */
	AM_RANGE(0xfff00000, 0xffffffff) AM_ROM AM_REGION("mb_bios", 0)     /* System BIOS */
ADDRESS_MAP_END

static ADDRESS_MAP_START(lindbergh_io, AS_IO, 32, lindbergh_state)
//	AM_IMPORT_FROM(pcat32_io_common)

//	AM_RANGE(0x00e8, 0x00ef) AM_NOP
//	AM_RANGE(0x0cf8, 0x0cff) AM_DEVREADWRITE("pcibus", pci_bus_legacy_device, read, write)
ADDRESS_MAP_END
#endif

lindbergh_state::lindbergh_state(const machine_config &mconfig, device_type type, const char *tag) : driver_device(mconfig, type, tag)
{
}

void lindbergh_state::machine_start()
{
}

void lindbergh_state::machine_reset()
{
}

static MACHINE_CONFIG_START(lindbergh, lindbergh_state)
//  MCFG_CPU_ADD("maincpu", PENTIUM, 2800000000U) /* Actually Celeron D at 2,8 GHz */
	MCFG_CPU_ADD("maincpu", PENTIUM4, 28000000U*5) /* Actually Celeron D at 2,8 GHz */
//	MCFG_CPU_PROGRAM_MAP(lindbergh_map)
//	MCFG_CPU_IO_MAP(lindbergh_io)
//	MCFG_CPU_IRQ_ACKNOWLEDGE_DEVICE("pic8259_1", pic8259_device, inta_cb)

//	MCFG_FRAGMENT_ADD( pcat_common )
//	MCFG_FRAGMENT_ADD( pcvideo_vga )

//	MCFG_PCI_BUS_LEGACY_ADD("pcibus", 0)

	MCFG_PCI_ROOT_ADD(                ":pci")
	MCFG_I82875P_HOST_ADD(            ":pci:00.0",                        0x103382c0, ":maincpu", 512*1024*1024)
	MCFG_I82875P_AGP_ADD(             ":pci:01.0")
	MCFG_GEFORCE_6800GT_ADD(          ":pci:01.0:00.0",                   0x10de0204)
	MCFG_PCI_BRIDGE_ADD(              ":pci:1c.0",      0x808625ae, 0x02)
	MCFG_I82541PI_ADD(                ":pci:1c.0:00.0",                   0x103382c0)
	MCFG_USB_UHCI_ADD(                ":pci:1d.0",      0x808625a9, 0x02, 0x103382c0)
	MCFG_USB_UHCI_ADD(                ":pci:1d.1",      0x808625aa, 0x02, 0x103382c0)
	MCFG_I6300ESB_WATCHDOG_ADD(       ":pci:1d.4",                        0x103382c0)
	MCFG_APIC_ADD(                    ":pci:1d.5",      0x808625ac, 0x02, 0x103382c0)
	MCFG_USB_EHCI_ADD(                ":pci:1d.7",      0x808625ad, 0x02, 0x103382c0)
	MCFG_PCI_BRIDGE_ADD(              ":pci:1e.0",      0x8086244e, 0x0a)
	MCFG_SB0400_ADD(                  ":pci:1e.0:02.0",                   0x11021101)
	MCFG_SEGA_LINDBERGH_BASEBOARD_ADD(":pci:1e.0:03.0")
	MCFG_I6300ESB_LPC_ADD(            ":pci:1f.0")
	MCFG_LPC_ACPI_ADD(                ":pci:1f.0:acpi")
	MCFG_LPC_RTC_ADD(                 ":pci:1f.0:rtc")
	MCFG_SATA_ADD(                    ":pci:1f.2",      0x808625a3, 0x02, 0x103382c0)
	MCFG_SMBUS_ADD(                   ":pci:1f.3",      0x808625a4, 0x02, 0x103382c0)
	MCFG_AC97_ADD(                    ":pci:1f.5",      0x808625a6, 0x02, 0x103382c0)
MACHINE_CONFIG_END

ROM_START(lindbios)
	ROM_REGION32_LE(0x100000, ":pci:1f.0", 0) // PC bios, location 3j7
	ROM_SYSTEM_BIOS(0, "bios0", "6.0.0010 alternate version")
	ROMX_LOAD("6.0.0010a.bin", 0x00000, 0x100000, CRC(10dd9b76) SHA1(1fdf1f921bc395846a7c3180fbdbc4ca287a9670), ROM_BIOS(1) )
	ROM_SYSTEM_BIOS(1, "bios1", "6.0.0009")
	ROMX_LOAD("6.0.0009.bin", 0x00000, 0x100000, CRC(5ffdfbf8) SHA1(605bc4967b749b4e6d13fc2ebb845ba956a259a7), ROM_BIOS(2) )
	ROM_SYSTEM_BIOS(2, "bios2", "6.0.0010")
	ROMX_LOAD("6.0.0010.bin", 0x00000, 0x100000, CRC(ea2bf888) SHA1(c9c5b6f0d4f4f36620939b15dd2f128a74347e37), ROM_BIOS(3) )


	ROM_REGION(0x400000, ":pci:1e.0:03.0", 0) // Baseboard MPC firmware
	ROM_LOAD("fpr-24370b.ic6", 0x000000, 0x400000, CRC(c3b021a4) SHA1(1b6938a50fe0e4ae813864649eb103838c399ac0))

	ROM_REGION32_LE(0x10000, ":pci:01.0:00.0", 0) // Geforce bios extension (custom or standard?)
	ROM_LOAD("vid_bios.u504", 0x00000, 0x10000, CRC(f78d14d7) SHA1(f129787e487984edd23bf344f2e9500c85052275))
ROM_END

GAME(1999, lindbios, 0, lindbergh, 0, driver_device, 0, ROT0, "Sega Lindbergh", "Sega Lindbergh Bios", GAME_IS_SKELETON)
_TIME(ATTOSECONDS_IN_USEC(2500)) MCFG_SCREEN_SIZE(40 * 12, 25 * 20) MCFG_SCREEN_VISIBLE_AREA(0, 40 * 12 - 1, 0, 25 * 20 - 1) MCFG_SCREEN_UPDATE_DEVICE("saa5050", saa5050_device, screen_update) MCFG_PALETTE_ADD("palette", 8) /* sound hardware */ MCFG_SPEAKER_STANDARD_MONO("mono") MCFG_SOUND_WAVE_ADD(WAVE_TAG, "cassette") MCFG_SOUND_ROUTE(ALL_OUTPUTS, "mono", 0.25) /* Devices */ MCFG_DEVICE_ADD("via", VIA6522, XTAL_4MHz / 4) MCFG_VIA6522_CB2_HANDLER(WRITELINE(a6809_state, cass_w)) MCFG_VIA6522_IRQ_HANDLER(DEVWRITELINE("maincpu", m6809e_device, irq_line)) MCFG_MC6845_ADD("mc6845", MC6845, "screen", XTAL_4MHz / 2, a6809_crtc6845_interface) MCFG_SAA5050_ADD("saa5050", 6000000, a6809_saa5050_intf) MCFG_ASCII_KEYBOARD_ADD("keyboard", kb_intf) MCFG_CASSETTE_ADD( "cassette", default_cassette_interface ) MCFG_TIMER_DRIVER_ADD_PERIODIC("a6809_c", a6809_state, a6809_c, attotime::from_hz(4800)) MCFG_TIMER_DRIVER_ADD_PERIODIC("a6809_p", a6809_state, a6809_p, attotime::from_hz(40000)) MACHINE_CONFIG_END /* ROM definition */ ROM_START( a6809 ) ROM_REGION( 0x800, "maincpu", 0 ) ROM_LOAD( "acorn6809.bin", 0x000, 0x800, CRC(5fa5b632) SHA1(b14a884bf82a7a8c23bc03c2e112728dd1a74896) ) ROM_REGION( 0x100, "proms", 0 ) ROM_LOAD( "acorn6809.ic11", 0x0000, 0x0100, CRC(7908317d) SHA1(e0f1e5bd3a8598d3b62bc432dd1f3892ed7e66d8) ) // address decoder ROM_END /* Driver */ /* YEAR NAME PARENT COMPAT MACHINE INPUT CLASS INIT COMPANY FULLNAME FLAGS */ COMP( 1980, a6809, 0, 0, a6809, a6809, driver_device, 0, "Acorn", "System 3 (6809 CPU)", 0 ) a> 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367
// license:BSD-3-Clause
// copyright-holders:Vas Crabb
/***************************************************************************

    srcclean.cpp

    Basic source code cleanear.

****************************************************************************/

/*
    Known general limitations:
    * Always uses filename.orig as backup location, and attempts to
      overwrite if it exists (doesn't try to generate unique name)
    * Assumes any input is UTF-8
    * No way to override hard-coded internal extension to syntax mapping
    * All Unicode characters are treated as occupying a single column
      (doesn't account for combining, non-spacing, fullwidth, etc.)

    Known C++ limitations:
    * No filtering of control characters
    * Will not produce expected output for a string continuation within
      a preprocessor macro, e.g this:
      #define MY_MACRO \
              "string that \
              continues"
    * Numeric literals broken by line continuations are not recognised
    * Will not recognise a comment delimiter broken by multiple line
      continuations. e.g. this:
      /\
      \
      / preprocessor abuse

    Known Lua limitations:
    * Whitespace normalisation is applied inside long string literals
      which can cause changes in behaviour
    * Disabled code inside long comments gets no special treatment and
      may have spacing adjusted in a way that affects behaviour when
      uncommented

    Known JSON limitations:
    * Doesn't detect invalid numbers or literals

    Known XML limitations:
    * No special handling for CDATA
    * No special handling for processing instructions
    * Doesn't do any kind of validation of structure
    * Doesn't do anything special for illegal -- in comment

    Features not carried over from previous version:
    * Stripping empty continuation lines
    * Stripping empty lines following open brace
*/

#include "corefile.h"
#include "corestr.h"
#include "strformat.h"

#include <algorithm>
#include <cassert>
#include <cstdint>
#include <cstdio>
#include <cstring>
#include <deque>
#include <fstream>
#include <functional>
#include <iostream>
#include <iterator>
#include <limits>
#include <memory>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>



namespace {

/***************************************************************************
    SOURCE CLEANER BASE CLASS
***************************************************************************/

class cleaner_base
{
public:
	enum class newline
	{
		DOS,
		UNIX,
		MACINTOSH,
		VMS
	};

	virtual ~cleaner_base() = default;

	template <typename InputIt>
	void process(InputIt begin, InputIt end);
	void finalise();

	virtual bool affected() const;
	virtual void summarise(std::ostream &os) const;

protected:
	static constexpr char32_t HORIZONTAL_TAB            = 0x0000'0009U;
	static constexpr char32_t LINE_FEED                 = 0x0000'000aU;
	static constexpr char32_t VERTICAL_TAB              = 0x0000'000bU;
	static constexpr char32_t FORM_FEED                 = 0x0000'000cU;
	static constexpr char32_t C0_CONTROL_LAST           = 0x0000'001fU;
	static constexpr char32_t SPACE                     = 0x0000'0020U;
	static constexpr char32_t DOUBLE_QUOTE              = 0x0000'0022U;
	static constexpr char32_t SINGLE_QUOTE              = 0x0000'0027U;
	static constexpr char32_t ASTERISK                  = 0x0000'002aU;
	static constexpr char32_t HYPHEN_MINUS              = 0x0000'002dU;
	static constexpr char32_t SLASH                     = 0x0000'002fU;
	static constexpr char32_t QUESTION_MARK             = 0x0000'003fU;
	static constexpr char32_t BACKSLASH                 = 0x0000'005cU;
	static constexpr char32_t BASIC_LATIN_LAST          = 0x0000'007fU;
	static constexpr char32_t CYRILLIC_SUPPLEMENT_LAST  = 0x0000'052fU;
	static constexpr char32_t LINE_SEPARATOR            = 0x0000'2028U;
	static constexpr char32_t PARAGRAPH_SEPARATOR       = 0x0000'2029U;

	template <typename OutputIt>
	cleaner_base(OutputIt &&output, newline newline_mode, unsigned tab_width);

	void output_character(char32_t ch);

	void set_tab_limit();
	void reset_tab_limit();

private:
	static constexpr char32_t CARRIAGE_RETURN       = 0x0000'000dU;
	static constexpr char32_t HIGH_SURROGATE_FIRST  = 0x0000'd800U;
	static constexpr char32_t HIGH_SURROGATE_LAST   = 0x0000'dbffU;
	static constexpr char32_t LOW_SURROGATE_FIRST   = 0x0000'dc00U;
	static constexpr char32_t LOW_SURROGATE_LAST    = 0x0000'dfffU;
	static constexpr char32_t NONCHARACTER_FIRST    = 0x0000'fdd0U;
	static constexpr char32_t NONCHARACTER_LAST     = 0x0000'fdefU;
	static constexpr char32_t ZERO_WIDTH_NB_SPACE   = 0x0000'feffU;
	static constexpr char32_t REPLACEMENT_CHARACTER = 0x0000'fffdU;
	static constexpr char32_t SUPPLEMENTARY_FIRST   = 0x0001'0000U;
	static constexpr char32_t SUPPLEMENTARY_LAST    = 0x0010'ffffU;

	static constexpr char32_t CODE_LENGTH_THRESHOLDS[6]{
			0x0000'0000U, 0x0000'0080U, 0x0000'0800U, 0x0001'0000U, 0x0020'0000U, 0x0400'0000 };

	typedef std::function<void (char)> output_function;

	virtual void process_characters(char32_t const *begin, char32_t const *end) = 0;
	virtual void input_complete() = 0;

	void flush_whitespace();
	void output_utf8(char32_t ch);
	void commit_character(char32_t ch);
	void process_if_full();
	void handle_lead_byte(std::uint8_t ch);
	void handle_codepoint(char32_t cp);

	static constexpr bool is_character(char32_t ch)
	{
		return
				(ch <= SUPPLEMENTARY_LAST) &&
				((ch < NONCHARACTER_FIRST) || (ch > NONCHARACTER_LAST)) &&
				((ch & 0x0000'fffeU) != 0x0000'fffeU);
	}

	static constexpr bool is_high_surrogate(char32_t ch)
	{
		return (ch >= HIGH_SURROGATE_FIRST) && (ch <= HIGH_SURROGATE_LAST);
	}

	static constexpr bool is_low_surrogate(char32_t ch)
	{
		return (ch >= LOW_SURROGATE_FIRST) && (ch <= LOW_SURROGATE_LAST);
	}

	static constexpr char32_t combine_surrogates(char32_t high, char32_t low)
	{
		return SUPPLEMENTARY_FIRST + (((high & 0x0000'03ffU) << 10U) | (low & 0x0000'03ffU));
	}

	// configuration
	newline         m_newline_mode;
	unsigned        m_tab_width;
	output_function m_output;

	// output state management
	unsigned                    m_output_column = 0U;
	unsigned                    m_indent;
	unsigned                    m_tab_limit     = std::numeric_limits<unsigned>::max();
	std::vector<char32_t>       m_whitespace;

	// input state management
	char32_t        m_buffer[1024];
	bool            m_stream_start      = true;
	std::size_t     m_position          = 0U;
	char32_t        m_surrogate         = 0U;
	unsigned        m_code_length       = 0U;
	unsigned        m_required_bytes    = 0U;
	char32_t        m_newline_lead      = 0U;

	// statistics
	std::uint64_t   m_overlong              = 0U;
	std::uint64_t   m_incomplete            = 0U;
	std::uint64_t   m_continuations         = 0U;
	std::uint64_t   m_invalid_bytes         = 0U;
	std::uint64_t   m_noncharacters         = 0U;
	std::uint64_t   m_surrogate_pairs       = 0U;
	std::uint64_t   m_lone_high_surrogates  = 0U;
	std::uint64_t   m_lone_low_surrogates   = 0U;
	std::uint64_t   m_leading_zw_nb_sp      = 0U;
	std::uint64_t   m_dos_newlines          = 0U;
	std::uint64_t   m_unix_newlines         = 0U;
	std::uint64_t   m_macintosh_newlines    = 0U;
	std::uint64_t   m_vms_newlines          = 0U;
	std::uint64_t   m_trailing_whitespace   = 0U;
	std::uint64_t   m_tabs_expanded         = 0U;
	std::uint64_t   m_tabs_created          = 0U;
	std::uint64_t   m_spaces_combined       = 0U;
	bool            m_final_newline         = false;
};

constexpr char32_t cleaner_base::CODE_LENGTH_THRESHOLDS[6];


/*--------------------------------------------------
    cleaner_base::process
    process a block of input bytes
--------------------------------------------------*/

template <typename InputIt>
void cleaner_base::process(InputIt begin, InputIt end)
{
	while (begin != end)
	{
		std::uint8_t const byte(*begin++);
		if (m_required_bytes)
		{
			if ((byte & 0xc0U) == 0x80U)
			{
				m_buffer[m_position] <<= 6U;
				m_buffer[m_position] |= char32_t(byte & 0x3fU);
				--m_required_bytes;
			}
			else
			{
				m_required_bytes = 0U;
				++m_incomplete;
				commit_character(REPLACEMENT_CHARACTER);
				handle_lead_byte(byte);
			}
		}
		else
		{
			handle_lead_byte(byte);
		}

		if (!m_required_bytes)
			handle_codepoint(m_buffer[m_position]);
	}
}


/*--------------------------------------------------
    cleaner_base::finalise
    perform final processing on reaching end of
    input
--------------------------------------------------*/

void cleaner_base::finalise()
{
	if (m_surrogate)
	{
		++m_lone_high_surrogates;
		commit_character(REPLACEMENT_CHARACTER);
		m_surrogate = 0U;
	}

	if (m_required_bytes)
	{
		++m_incomplete;
		commit_character(REPLACEMENT_CHARACTER);
	}

	switch (m_newline_lead)
	{
	case LINE_FEED:
		++m_unix_newlines;
		m_newline_lead = 0U;
		m_buffer[m_position++] = LINE_FEED;
		break;
	case CARRIAGE_RETURN:
		++m_macintosh_newlines;
		m_newline_lead = 0U;
		m_buffer[m_position++] = LINE_FEED;
		break;
	default:
		assert(!m_newline_lead);
	}

	if (m_position)
	{
		process_characters(m_buffer, m_buffer + m_position);
		m_position = 0U;
	}

	input_complete();

	if (m_output_column || !m_whitespace.empty())
	{
		m_final_newline = true;
		output_character(LINE_FEED);
	}
}


/*--------------------------------------------------
    cleaner_base::affected
    returns whether any cleanups have been
    applied
--------------------------------------------------*/

bool cleaner_base::affected() const
{
	return
			m_overlong ||
			m_incomplete ||
			m_continuations ||
			m_invalid_bytes ||
			m_noncharacters ||
			m_surrogate_pairs ||
			m_lone_high_surrogates ||
			m_lone_low_surrogates ||
			m_leading_zw_nb_sp ||
			(m_dos_newlines && (newline::DOS != m_newline_mode)) ||
			(m_unix_newlines && (newline::UNIX != m_newline_mode)) ||
			(m_macintosh_newlines && (newline::MACINTOSH != m_newline_mode)) ||
			(m_vms_newlines && (newline::VMS != m_newline_mode)) ||
			m_trailing_whitespace ||
			m_tabs_expanded ||
			m_tabs_created ||
			m_spaces_combined ||
			m_final_newline;
}


/*--------------------------------------------------
    cleaner_base::summarise
    print summary of changes applied
--------------------------------------------------*/

void cleaner_base::summarise(std::ostream &os) const
{
	if (m_overlong)
		util::stream_format(os, "%1$u overlong UTF-8 sequence(s) corrected\n", m_overlong);
	if (m_incomplete)
		util::stream_format(os, "%1$u incomplete UTF-8 sequence(s) replaced\n", m_incomplete);
	if (m_continuations)
		util::stream_format(os, "%1$u UTF-8 continuation(s) replaced\n", m_continuations);
	if (m_invalid_bytes)
		util::stream_format(os, "%1$u invalid UTF-8 byte(s) replaced\n", m_invalid_bytes);
	if (m_noncharacters)
		util::stream_format(os, "%1$u noncharacter(s) replaced\n", m_noncharacters);
	if (m_surrogate_pairs)
		util::stream_format(os, "%1$u surrogate pair(s) combined\n", m_surrogate_pairs);
	if (m_lone_high_surrogates)
		util::stream_format(os, "%1$u lone high surrogate(s) replaced\n", m_lone_high_surrogates);
	if (m_lone_low_surrogates)
		util::stream_format(os, "%1$u lone low surrogate(s) replaced\n", m_lone_low_surrogates);
	if (m_leading_zw_nb_sp)
		util::stream_format(os, "%1$u leading zero-width no-break space(s) removed\n", m_leading_zw_nb_sp);
	if (m_dos_newlines && (newline::DOS != m_newline_mode))
		util::stream_format(os, "%1$u DOS line ending(s) normalised\n", m_dos_newlines);
	if (m_unix_newlines && (newline::UNIX != m_newline_mode))
		util::stream_format(os, "%1$u UNIX line ending(s) normalised\n", m_unix_newlines);
	if (m_macintosh_newlines && (newline::MACINTOSH != m_newline_mode))
		util::stream_format(os, "%1$u Macintosh line ending(s) normalised\n", m_macintosh_newlines);
	if (m_vms_newlines && (newline::VMS != m_newline_mode))
		util::stream_format(os, "%1$u VMS line ending(s) normalised\n", m_vms_newlines);
	if (m_trailing_whitespace)
		util::stream_format(os, "%1$u line(s) with trailing whitespace trimmed\n", m_trailing_whitespace);
	if (m_tabs_expanded)
		util::stream_format(os, "%1$u tab(s) expanded to spaces\n", m_tabs_expanded);
	if (m_tabs_created)
		util::stream_format(os, "%1$u tab(s) created from spaces\n", m_tabs_created);
	if (m_spaces_combined)
		util::stream_format(os, "%1$u space(s) combined into tabs\n", m_spaces_combined);
	if (m_final_newline)
		util::stream_format(os, "line ending added at end of file\n");
}


/*--------------------------------------------------
    cleaner_base::cleaner_base
    base constructor
--------------------------------------------------*/

template <typename OutputIt>
cleaner_base::cleaner_base(
		OutputIt &&output,
		newline newline_mode,
		unsigned tab_width)
	: m_newline_mode(newline_mode)
	, m_tab_width(tab_width)
	, m_output([it = std::forward<OutputIt>(output)] (char ch) mutable { *it++ = ch; })
	, m_whitespace()
{
	m_whitespace.reserve(128U);
}


/*--------------------------------------------------
    cleaner_base::output_character
    output character applying whitespace
    normalisation and line ending translation
--------------------------------------------------*/

void cleaner_base::output_character(char32_t ch)
{
	switch (ch)
	{
	case HORIZONTAL_TAB:
	case SPACE:
		m_whitespace.emplace_back(ch);
		break;

	case LINE_FEED:
		m_output_column = 0U;
		if (!m_whitespace.empty())
		{
			++m_trailing_whitespace;
			m_whitespace.clear();
		}
		switch (m_newline_mode)
		{
		case newline::DOS:
			output_utf8(CARRIAGE_RETURN);
			output_utf8(LINE_FEED);
			break;
		case newline::UNIX:
			output_utf8(LINE_FEED);
			break;
		case newline::MACINTOSH:
			output_utf8(CARRIAGE_RETURN);
			break;
		case newline::VMS:
			output_utf8(LINE_FEED);
			output_utf8(CARRIAGE_RETURN);
			break;
		}
		break;

	default:
		flush_whitespace();
		++m_output_column;
		output_utf8(ch);
	}
}


/*--------------------------------------------------
    cleaner_base::set_tab_limit
    limit leading tabs to number used to indent
    current line
--------------------------------------------------*/

void cleaner_base::set_tab_limit()
{
	if (!m_output_column)
	{
		unsigned limit(0U);
		for (char32_t ch : m_whitespace)
			limit += (HORIZONTAL_TAB == ch) ? (m_tab_width - (limit % m_tab_width)) : 1U;
		m_tab_limit = limit;
	}
	else
	{
		m_tab_limit = m_indent;
	}
}


/*--------------------------------------------------
    cleaner_base::reset_tab_limit
    revert to default handling of leading tabs
--------------------------------------------------*/

void cleaner_base::reset_tab_limit()
{
	m_tab_limit = std::numeric_limits<unsigned>::max();
}


/*--------------------------------------------------
    cleaner_base::flush_whitespace
    send whitespace to output normalising spaces
    and tabs in initial indent
--------------------------------------------------*/

void cleaner_base::flush_whitespace()
{
	bool const  set_indent(!m_output_column);
	bool        expand(m_output_column);
	unsigned    space_count(0U);
	for (char32_t space : m_whitespace)
	{
		assert(!expand || !space_count);
		assert(space_count < m_tab_width);

		if (HORIZONTAL_TAB == space)
		{
			unsigned width(m_tab_width - (m_output_column % m_tab_width));
			expand = expand || ((width + m_output_column) > m_tab_limit);
			if (expand)
			{
				++m_tabs_expanded;
				while (width--)
				{
					++m_output_column;
					output_utf8(SPACE);
				}
			}
			else
			{
				assert(!(m_output_column % m_tab_width));

				m_spaces_combined += space_count;
				m_output_column += width;
				output_utf8(space);
			}
			space_count = 0U;
		}
		else
		{
			assert(SPACE == space);

			++space_count;
			expand = expand || ((space_count + m_output_column) > m_tab_limit);
			if (expand)
			{
				while (space_count)
				{
					space_count--;
					++m_output_column;
					output_utf8(SPACE);
				}
			}
			else
			{
				assert(!(m_output_column % m_tab_width));

				if (space_count == m_tab_width)
				{
					++m_tabs_created;
					m_spaces_combined += space_count;
					space_count = 0U;
					m_output_column += m_tab_width;
					output_utf8(HORIZONTAL_TAB);
				}
			}
		}
	}
	while (space_count--)
	{
		++m_output_column;
		output_utf8(SPACE);
	}
	m_whitespace.clear();
	if (set_indent)
		m_indent = m_output_column;
}


/*--------------------------------------------------
    cleaner_base::output_utf8
    convert codepoint to UFF-8 and send to output
--------------------------------------------------*/

void cleaner_base::output_utf8(char32_t ch)
{
	if (0x0000'0080U > ch)
	{
		m_output(char(std::uint8_t(ch >> 0U)));
	}
	else
	{
		unsigned required =
				(0x0000'0800U > ch) ? 1U :
				(0x0001'0000U > ch) ? 2U :
				(0x0020'0000U > ch) ? 3U :
				(0x0400'0000U > ch) ? 4U : 5U;
		m_output(char(std::uint8_t(((ch >> (6U * required)) & (0x3fU >> required)) | ((0xfcU << (5U - required)) & 0xfcU))));
		while (required--)
			m_output(char(std::uint8_t(((ch >> (6U * required)) & 0x3fU) | 0x80U)));
	}
}


/*--------------------------------------------------
    cleaner_base::commit_character
    store decoded input character in buffer
    applying line ending normalisation and
    replacing noncharacters
--------------------------------------------------*/

void cleaner_base::commit_character(char32_t ch)
{
	assert(std::size(m_buffer) > m_position);
	assert(1U <= m_code_length);
	assert(6U >= m_code_length);

	if (CODE_LENGTH_THRESHOLDS[m_code_length - 1] > ch)
		++m_overlong;

	if (m_stream_start)
	{
		assert(!m_position);
		assert(!m_newline_lead);

		if (ZERO_WIDTH_NB_SPACE == ch)
		{
			++m_leading_zw_nb_sp;
			return;
		}
		else
		{
			m_stream_start = false;
		}
	}

	if (!is_character(ch))
	{
		ch = REPLACEMENT_CHARACTER;
		++m_noncharacters;
	}

	switch (ch)
	{
	case LINE_FEED:
		switch (m_newline_lead)
		{
		case LINE_FEED:
			++m_unix_newlines;
			m_buffer[m_position++] = LINE_FEED;
			break;
		case CARRIAGE_RETURN:
			++m_dos_newlines;
			m_newline_lead = 0U;
			m_buffer[m_position++] = LINE_FEED;
			break;
		default:
			assert(!m_newline_lead);
			m_newline_lead = ch;
		}
		break;

	case CARRIAGE_RETURN:
		switch (m_newline_lead)
		{
		case LINE_FEED:
			++m_vms_newlines;
			m_newline_lead = 0U;
			m_buffer[m_position++] = LINE_FEED;
			break;
		case CARRIAGE_RETURN:
			++m_macintosh_newlines;
			m_buffer[m_position++] = LINE_FEED;
			break;
		default:
			assert(!m_newline_lead);
			m_newline_lead = ch;
		}
		break;

	default:
		switch (m_newline_lead)
		{
		case LINE_FEED:
			++m_unix_newlines;
			m_newline_lead = 0U;
			m_buffer[m_position++] = LINE_FEED;
			process_if_full();
			break;
		case CARRIAGE_RETURN:
			++m_macintosh_newlines;
			m_newline_lead = 0U;
			m_buffer[m_position++] = LINE_FEED;
			process_if_full();
			break;
		default:
			assert(!m_newline_lead);
		};
		m_buffer[m_position++] = ch;
	}

	process_if_full();
}


/*--------------------------------------------------
    cleaner_base::process_if_full
    perform processing on decoded characters if
    buffer is full
--------------------------------------------------*/

void cleaner_base::process_if_full()
{
	if (std::size(m_buffer) == m_position)
	{
		process_characters(m_buffer, m_buffer + m_position);
		m_position = 0U;
	}
}


/*--------------------------------------------------
    cleaner_base::handle_lead_byte
    handle an input byte that isn't a valid UTF-8
    continuation
--------------------------------------------------*/

void cleaner_base::handle_lead_byte(std::uint8_t byte)
{
	m_required_bytes =
			((byte & 0xfeU) == 0xfcU) ? 5U :
			((byte & 0xfcU) == 0xf8U) ? 4U :
			((byte & 0xf8U) == 0xf0U) ? 3U :
			((byte & 0xf0U) == 0xe0U) ? 2U :
			((byte & 0xe0U) == 0xc0U) ? 1U : 0U;
	m_code_length = m_required_bytes + 1U;
	if (m_required_bytes)
	{
		m_buffer[m_position] = ((char32_t(1U) << (6U - m_required_bytes)) - 1) & char32_t(byte);
	}
	else if ((byte & 0xc0U) == 0x80U)
	{
		m_buffer[m_position] = REPLACEMENT_CHARACTER;
		++m_continuations;
	}
	else if ((byte & 0xfeU)  == 0xfeU)
	{
		m_buffer[m_position] = REPLACEMENT_CHARACTER;
		++m_invalid_bytes;
	}
	else
	{
		m_buffer[m_position] = byte;
	}
}


/*--------------------------------------------------
    cleaner_base::handle_codepoint
    handle a decoded UTF-8 unit dealing with
    surrogates
--------------------------------------------------*/

void cleaner_base::handle_codepoint(char32_t cp)
{
	if (m_surrogate)
	{
		if (is_low_surrogate(cp))
		{
			++m_surrogate_pairs;
			commit_character(combine_surrogates(m_surrogate, cp));
			m_surrogate = 0U;
		}
		else
		{
			++m_lone_high_surrogates;
			commit_character(REPLACEMENT_CHARACTER);
			m_surrogate = 0U;
			handle_codepoint(cp);
		}
	}
	else if (is_high_surrogate(cp))
	{
		m_surrogate = cp;
	}
	else if (is_low_surrogate(cp))
	{
		++m_lone_low_surrogates;
		commit_character(REPLACEMENT_CHARACTER);
	}
	else
	{
		commit_character(cp);
	}
}



/***************************************************************************
    PLAIN TEXT CLEANER CLASS
***************************************************************************/

class text_cleaner : public cleaner_base
{
public:
	template <typename OutputIt>
	text_cleaner(OutputIt &&output, newline newline_mode, unsigned tab_width)
		: cleaner_base(std::forward<OutputIt>(output), newline_mode, tab_width)
	{
	}

private:
	virtual void process_characters(char32_t const *begin, char32_t const *end) override
	{
		while (begin != end)
			output_character(*begin++);
	}

	virtual void input_complete() override
	{
	}
};



/***************************************************************************
    C++ SOURCE CLEANER CLASS
***************************************************************************/

class cpp_cleaner : public cleaner_base
{
public:
	template <typename OutputIt>
	cpp_cleaner(OutputIt &&output, newline newline_mode, unsigned tab_width);

	virtual bool affected() const override;
	virtual void summarise(std::ostream &os) const override;

protected:
	void output_character(char32_t ch);

private:
	static constexpr char32_t UPPERCASE_FIRST       = 0x0000'0041U;
	static constexpr char32_t UPPERCASE_B           = 0x0000'0042U;
	static constexpr char32_t UPPERCASE_X           = 0x0000'0058U;
	static constexpr char32_t UPPERCASE_LAST        = 0x0000'005aU;
	static constexpr char32_t UNDERSCORE            = 0x0000'005fU;
	static constexpr char32_t LOWERCASE_FIRST       = 0x0000'0061U;
	static constexpr char32_t LOWERCASE_B           = 0x0000'0062U;
	static constexpr char32_t LOWERCASE_X           = 0x0000'0078U;
	static constexpr char32_t LOWERCASE_LAST        = 0x0000'007aU;

	static constexpr char32_t DIGIT_FIRST           = 0x0000'0030U;
	static constexpr char32_t DIGIT_BINARY_LAST     = 0x0000'0031U;
	static constexpr char32_t DIGIT_OCTAL_LAST      = 0x0000'0037U;
	static constexpr char32_t DIGIT_DECIMAL_LAST    = 0x0000'0039U;
	static constexpr char32_t DIGIT_HEX_UPPER_FIRST = 0x0000'0041U;
	static constexpr char32_t DIGIT_HEX_UPPER_LAST  = 0x0000'0046U;
	static constexpr char32_t DIGIT_HEX_LOWER_FIRST = 0x0000'0061U;
	static constexpr char32_t DIGIT_HEX_LOWER_LAST  = 0x0000'0066U;

	enum class parse_state
	{
		DEFAULT,
		COMMENT,
		LINE_COMMENT,
		TOKEN,
		STRING_CONSTANT,
		CHARACTER_CONSTANT,
		NUMERIC_CONSTANT
	};

	virtual void process_characters(char32_t const *begin, char32_t const *end) override;
	virtual void input_complete() override;

	void process_default(char32_t ch);
	void process_comment(char32_t ch);
	void process_line_comment(char32_t ch);
	void process_token(char32_t ch);
	void process_text(char32_t ch);
	void process_numeric(char32_t ch);

	bool tail_is(char32_t ch) const
	{
		return !m_tail.empty() && (m_tail.back() == ch);
	}

	void pop_tail()
	{
		if (!m_tail.empty())
			m_tail.pop_back();
	}

	void replace_tail(char32_t ch)
	{
		assert(!m_tail.empty());
		m_tail.back() = ch;
	}

	void flush_tail()
	{
		for (char32_t tail : m_tail)
			cleaner_base::output_character(tail);
		m_tail.clear();
	}

	static constexpr bool is_token_lead(char32_t ch)
	{
		return
				((UPPERCASE_FIRST <= ch) && (UPPERCASE_LAST >= ch)) ||
				((LOWERCASE_FIRST <= ch) && (LOWERCASE_LAST >= ch)) ||
				(UNDERSCORE == ch);
	}

	static constexpr bool is_token_continuation(char32_t ch)
	{
		return
				is_token_lead(ch) ||
				((DIGIT_FIRST <= ch) && (DIGIT_DECIMAL_LAST >= ch));
	}

	static constexpr bool is_numeric_lead(char32_t ch)
	{
		return (DIGIT_FIRST <= ch) && (DIGIT_DECIMAL_LAST >= ch);
	}

	static constexpr bool is_binary_digit(char32_t ch)
	{
		return (DIGIT_FIRST <= ch) && (DIGIT_BINARY_LAST >= ch);
	}

	static constexpr bool is_octal_digit(char32_t ch)
	{
		return (DIGIT_FIRST <= ch) && (DIGIT_OCTAL_LAST >= ch);
	}

	static constexpr bool is_decimal_digit(char32_t ch)
	{
		return (DIGIT_FIRST <= ch) && (DIGIT_DECIMAL_LAST >= ch);
	}

	static constexpr bool is_hexadecimal_digit(char32_t ch)
	{
		return
				((DIGIT_FIRST <= ch) && (DIGIT_DECIMAL_LAST >= ch)) ||
				((DIGIT_HEX_UPPER_FIRST <= ch) && (DIGIT_HEX_UPPER_LAST >= ch)) ||
				((DIGIT_HEX_LOWER_FIRST <= ch) && (DIGIT_HEX_LOWER_LAST >= ch));
	}

	parse_state                 m_parse_state   = parse_state::DEFAULT;
	std::uint64_t               m_input_line    = 1U;
	bool                        m_escape        = false;
	std::deque<char32_t>        m_tail;
	std::uint64_t               m_comment_line  = 0U;
	bool                        m_broken_escape = false;
	char32_t                    m_lead_digit    = 0U;
	unsigned                    m_radix         = 0U;

	std::uint64_t   m_tabs_escaped                  = 0U;
	std::uint64_t   m_broken_comment_delimiters     = 0U;
	std::uint64_t   m_line_comment_continuations    = 0U;
	std::uint64_t   m_string_continuations          = 0U;
	std::uint64_t   m_uppercase_radix               = 0U;
	std::uint64_t   m_non_ascii                     = 0U;
};


template <typename OutputIt>
cpp_cleaner::cpp_cleaner(
		OutputIt &&output,
		newline newline_mode,
		unsigned tab_width)
	: cleaner_base(std::forward<OutputIt>(output), newline_mode, tab_width)
	, m_tail()
{
}


bool cpp_cleaner::affected() const
{
	return
			cleaner_base::affected() ||
			m_tabs_escaped ||
			m_broken_comment_delimiters ||
			m_line_comment_continuations ||
			m_string_continuations ||
			m_uppercase_radix ||
			m_non_ascii;
}


void cpp_cleaner::summarise(std::ostream &os) const
{
	cleaner_base::summarise(os);
	if (m_tabs_escaped)
		util::stream_format(os, "%1$u tab(s) escaped\n", m_tabs_escaped);
	if (m_broken_comment_delimiters)
		util::stream_format(os, "%1$u broken comment delimiter(s) replaced\n", m_broken_comment_delimiters);
	if (m_line_comment_continuations)
		util::stream_format(os, "%1$u line comment continuation(s) replaced\n", m_line_comment_continuations);
	if (m_string_continuations)
		util::stream_format(os, "%1$u string literal continuation(s) replaced\n", m_string_continuations);
	if (m_uppercase_radix)
		util::stream_format(os, "%1$u uppercase radix character(s) normalised\n", m_uppercase_radix);
	if (m_non_ascii)
		util::stream_format(os, "%1$u non-ASCII character(s) replaced\n", m_non_ascii);
}


void cpp_cleaner::output_character(char32_t ch)
{
	switch (m_parse_state)
	{
	case parse_state::DEFAULT:
	case parse_state::TOKEN:
	case parse_state::CHARACTER_CONSTANT:
	case parse_state::NUMERIC_CONSTANT:
		if (BASIC_LATIN_LAST < ch)
		{
			++m_non_ascii;
			ch = QUESTION_MARK;
		}
		break;
	case parse_state::COMMENT:
	case parse_state::LINE_COMMENT:
		break;
	case parse_state::STRING_CONSTANT:
		if (CYRILLIC_SUPPLEMENT_LAST < ch)
		{
			++m_non_ascii;
			ch = QUESTION_MARK;
		}
		break;
	}

	switch (ch)
	{
	case HORIZONTAL_TAB:
	case SPACE:
	case BACKSLASH:
		m_tail.emplace_back(ch);
		break;
	default:
		flush_tail();
		if (LINE_FEED == ch)
			cleaner_base::output_character(ch);
		else
			m_tail.emplace_back(ch);
	}
}


void cpp_cleaner::process_characters(char32_t const *begin, char32_t const *end)
{
	while (begin != end)
	{
		char32_t const ch(*begin++);
		switch (m_parse_state)
		{
		case parse_state::DEFAULT:
			process_default(ch);
			break;
		case parse_state::COMMENT:
			process_comment(ch);
			break;
		case parse_state::LINE_COMMENT:
			process_line_comment(ch);
			break;
		case parse_state::TOKEN:
			process_token(ch);
			break;
		case parse_state::CHARACTER_CONSTANT:
		case parse_state::STRING_CONSTANT:
			process_text(ch);
			break;
		case parse_state::NUMERIC_CONSTANT:
			process_numeric(ch);
			break;
		}

		if (LINE_FEED == ch)
			++m_input_line;
	}
}


void cpp_cleaner::input_complete()
{
	flush_tail();
	switch (m_parse_state)
	{
	case parse_state::COMMENT:
		throw std::runtime_error(util::string_format("unterminated multi-line comment beginning on line %1$u", m_comment_line));
	case parse_state::CHARACTER_CONSTANT:
		throw std::runtime_error(util::string_format("unterminated character literal on line %1$u", m_input_line));
	case parse_state::STRING_CONSTANT:
		throw std::runtime_error(util::string_format("unterminated string literal on line %1$u", m_input_line));
	default:
		break;
	}
}


void cpp_cleaner::process_default(char32_t ch)
{
	switch (ch)
	{
	case LINE_FEED:
		if (m_escape && tail_is(BACKSLASH))
		{
			m_broken_escape = true;
			return;
		}
		break;
	case DOUBLE_QUOTE:
		m_parse_state = parse_state::STRING_CONSTANT;
		break;
	case SINGLE_QUOTE:
		m_parse_state = parse_state::CHARACTER_CONSTANT;
		break;
	case ASTERISK:
		if (m_escape)
		{
			m_parse_state = parse_state::COMMENT;
			m_comment_line = m_input_line;
			set_tab_limit();
			if (m_broken_escape)
			{
				++m_broken_comment_delimiters;
				assert(tail_is(BACKSLASH));
				pop_tail();
				output_character(ch);
				output_character(LINE_FEED);
				m_escape = false;
				m_broken_escape = false;
				return;
			}
		}
		break;
	case SLASH:
		if (m_escape)
		{
			m_parse_state = parse_state::LINE_COMMENT;
			if (m_broken_escape)
			{
				++m_broken_comment_delimiters;
				assert(tail_is(BACKSLASH));
				pop_tail();
				assert(tail_is(SLASH));
				pop_tail();
				output_character(LINE_FEED);
				output_character(SLASH);
				m_broken_escape = false;
			}
		}
		break;
	default:
		if (is_token_lead(ch))
		{
			m_parse_state = parse_state::TOKEN;
		}
		else if (is_numeric_lead(ch))
		{
			m_parse_state = parse_state::NUMERIC_CONSTANT;
			m_escape = false;
			m_broken_escape = false;
			process_numeric(ch);
			return;
		}
	}
	if (m_broken_escape)
		output_character(LINE_FEED);
	m_escape = m_escape ? ((BACKSLASH == ch) && tail_is(SLASH)) : (SLASH == ch);
	m_broken_escape = false;
	output_character(ch);
}


void cpp_cleaner::process_comment(char32_t ch)
{
	switch (ch)
	{
	case LINE_FEED:
		if (m_escape && tail_is(BACKSLASH))
		{
			m_broken_escape = true;
		}
		else
		{
			m_escape = false;
			m_broken_escape = false;
			output_character(ch);
		}
		break;
	case SLASH:
		if (m_broken_escape)
		{
			m_parse_state = parse_state::DEFAULT;
			m_comment_line = 0U;
			++m_broken_comment_delimiters;
			assert(tail_is(BACKSLASH));
			pop_tail();
			assert(tail_is(ASTERISK));
			pop_tail();
			output_character(LINE_FEED);
			output_character(ASTERISK);
			output_character(ch);
			reset_tab_limit();
		}
		else if (m_escape)
		{
			m_parse_state = parse_state::DEFAULT;
			m_comment_line = 0U;
			output_character(ch);
			reset_tab_limit();
		}
		else
		{
			output_character(ch);
		}
		m_escape = false;
		m_broken_escape = false;
		break;
	case BACKSLASH:
		if (m_broken_escape)
		{
			m_escape = false;
			m_broken_escape = false;
			output_character(LINE_FEED);
		}
		else if (m_escape)
		{
			m_escape = tail_is(ASTERISK);
		}
		output_character(ch);
		break;
	default:
		if (m_broken_escape)
			output_character(LINE_FEED);
		m_escape = ASTERISK == ch;
		m_broken_escape = false;
		output_character(ch);
	}
}


void cpp_cleaner::process_line_comment(char32_t ch)
{
	switch (ch)
	{
	case LINE_FEED:
		if (tail_is(BACKSLASH))
		{
			++m_line_comment_continuations;
			pop_tail();
			output_character(ch);
			output_character(SLASH);
			output_character(SLASH);
			break;
		}
		m_parse_state = parse_state::DEFAULT;
		[[fallthrough]];
	default:
		output_character(ch);
	}
}


void cpp_cleaner::process_token(char32_t ch)
{
	if (is_token_continuation(ch))
	{
		output_character(ch);
	}
	else
	{
		m_parse_state = parse_state::DEFAULT;
		process_default(ch);
	}
}


void cpp_cleaner::process_text(char32_t ch)
{
	switch (ch)
	{
	case HORIZONTAL_TAB:
		++m_tabs_escaped;
		if (!m_escape)
			output_character(BACKSLASH);
		output_character(char32_t(std::uint8_t('t')));
		break;
	case LINE_FEED:
		if (parse_state::CHARACTER_CONSTANT == m_parse_state)
		{
			throw std::runtime_error(util::string_format("unterminated character literal on line %1$u", m_input_line));
		}
		else if (tail_is(BACKSLASH))
		{
			++m_string_continuations;
			if (m_escape)
			{
				replace_tail(DOUBLE_QUOTE);
				output_character(ch);
				output_character(DOUBLE_QUOTE);
			}
			else
			{
				pop_tail();
				assert(tail_is(BACKSLASH));
				pop_tail();
				output_character(DOUBLE_QUOTE);
				output_character(ch);
				output_character(DOUBLE_QUOTE);
				output_character(BACKSLASH);
				m_escape = true;
				return;
			}
		}
		else
		{
			throw std::runtime_error(util::string_format("unterminated string literal on line %1$u", m_input_line));
		}
		break;
	case VERTICAL_TAB:
		++m_tabs_escaped;
		if (!m_escape)
			output_character(BACKSLASH);
		output_character(char32_t(std::uint8_t('v')));
		break;
	default:
		output_character(ch);
		if (!m_escape && (((parse_state::STRING_CONSTANT == m_parse_state) ? DOUBLE_QUOTE : SINGLE_QUOTE) == ch))
			m_parse_state = parse_state::DEFAULT;
	}
	m_escape = (BACKSLASH == ch) && !m_escape;
}


void cpp_cleaner::process_numeric(char32_t ch)
{
	if (!m_lead_digit)
	{
		assert(is_numeric_lead(ch));
		assert(!m_radix);

		m_lead_digit = ch;
		if (DIGIT_FIRST != ch)
			m_radix = 10U;
	}
	else if (!m_radix)
	{
		assert(DIGIT_FIRST == m_lead_digit);

		switch (ch)
		{
		case SINGLE_QUOTE:
			if (m_escape)
				throw std::runtime_error(util::string_format("adjacent digit separators on line %1$u", m_input_line));
			else
				m_escape = true;
			break;
		case UPPERCASE_B:
			++m_uppercase_radix;
			ch = LOWERCASE_B;
			[[fallthrough]];
		case LOWERCASE_B:
			m_radix = 2U;
			break;
		case UPPERCASE_X:
			++m_uppercase_radix;
			ch = LOWERCASE_X;
			[[fallthrough]];
		case LOWERCASE_X:
			m_radix = 16U;
			break;
		default:
			if (is_octal_digit(ch))
				m_radix = 8U;
			else if (is_decimal_digit(ch))
				m_parse_state = parse_state::DEFAULT; // this should be an invalid octal literal, but it's probably just an argument to the SHA1 macro
			else
				m_parse_state = parse_state::DEFAULT;
		}
	}
	else
	{
		if (SINGLE_QUOTE == ch)
		{
			if (m_escape)
				throw std::runtime_error(util::string_format("adjacent digit separators on line %1$u", m_input_line));
			else
				m_escape = true;
		}
		else
		{
			m_escape = false;
			switch (m_radix)
			{
			case 2U:
				if (!is_decimal_digit(ch))
					m_parse_state = parse_state::DEFAULT;
				else if (!is_binary_digit(ch))
					m_parse_state = parse_state::DEFAULT; // this should be an invalid binary literal, but it's probably just an argument to the SHA1 macro
				break;
			case 8U:
				if (!is_decimal_digit(ch))
					m_parse_state = parse_state::DEFAULT;
				else if (!is_octal_digit(ch))
					m_parse_state = parse_state::DEFAULT; // this should be an invalid octal literal, but it's probably just an argument to the SHA1 macro
				break;
			case 10U:
				if (!is_decimal_digit(ch))
					m_parse_state = parse_state::DEFAULT;
				break;
			case 16U:
				if (!is_hexadecimal_digit(ch))
					m_parse_state = parse_state::DEFAULT;
				break;
			default:
				assert(false);
				m_parse_state = parse_state::DEFAULT;
			}
		}
	}

	if (parse_state::DEFAULT == m_parse_state)
	{
		m_escape = false;
		m_lead_digit = 0U;
		m_radix = 0U;
		process_default(ch);
	}
	else
	{
		assert(parse_state::NUMERIC_CONSTANT == m_parse_state);

		output_character(ch);
	}
}



/***************************************************************************
    LUA SOURCE CLEANER CLASS
***************************************************************************/

class lua_cleaner : public cleaner_base
{
public:
	template <typename OutputIt>
	lua_cleaner(OutputIt &&output, newline newline_mode, unsigned tab_width);

	virtual bool affected() const override;
	virtual void summarise(std::ostream &os) const override;

protected:
	void output_character(char32_t ch);

private:
	static constexpr char32_t EQUALS        = 0x0000'003dU;
	static constexpr char32_t LEFT_BRACKET  = 0x0000'005bU;
	static constexpr char32_t RIGHT_BRACKET = 0x0000'005dU;

	enum class parse_state
	{
		DEFAULT,
		SHORT_COMMENT,
		LONG_COMMENT,
		STRING_CONSTANT,
		LONG_STRING_CONSTANT
	};

	virtual void process_characters(char32_t const *begin, char32_t const *end) override;
	virtual void input_complete() override;

	void process_default(char32_t ch);
	void process_short_comment(char32_t ch);
	void process_long_comment(char32_t ch);
	void process_string_constant(char32_t ch);
	void process_long_string_constant(char32_t ch);

	parse_state     m_parse_state           = parse_state::DEFAULT;
	std::uint64_t   m_input_line            = 1U;
	int             m_long_bracket_level    = -1;
	bool            m_escape                = false;
	std::uint32_t   m_block_line            = 0U;
	int             m_block_level           = 0;
	bool            m_comment_start         = false;
	char32_t        m_string_quote          = 0U;

	std::uint64_t   m_tabs_escaped      = 0U;
	std::uint64_t   m_newlines_escaped  = 0U;
	std::uint64_t   m_non_ascii         = 0U;
};


template <typename OutputIt>
lua_cleaner::lua_cleaner(
		OutputIt &&output,
		newline newline_mode,
		unsigned tab_width)
	: cleaner_base(std::forward<OutputIt>(output), newline_mode, tab_width)
{
}


bool lua_cleaner::affected() const
{
	return
			cleaner_base::affected() ||
			m_tabs_escaped ||
			m_newlines_escaped ||
			m_non_ascii;
}


void lua_cleaner::summarise(std::ostream &os) const
{
	cleaner_base::summarise(os);
	if (m_tabs_escaped)
		util::stream_format(os, "%1$u tab(s) escaped\n", m_tabs_escaped);
	if (m_newlines_escaped)
		util::stream_format(os, "%1$u escaped line ending(s) converted\n", m_newlines_escaped);
	if (m_non_ascii)
		util::stream_format(os, "%1$u non-ASCII character(s) replaced\n", m_non_ascii);
}


void lua_cleaner::output_character(char32_t ch)
{
	switch (m_parse_state)
	{
	case parse_state::DEFAULT:
		if (BASIC_LATIN_LAST < ch)
		{
			++m_non_ascii;
			ch = QUESTION_MARK;
		}
		break;
	case parse_state::SHORT_COMMENT:
	case parse_state::LONG_COMMENT:
		break;
	case parse_state::STRING_CONSTANT:
	case parse_state::LONG_STRING_CONSTANT:
		if (CYRILLIC_SUPPLEMENT_LAST < ch)
		{
			++m_non_ascii;
			ch = QUESTION_MARK;
		}
		break;
	}

	cleaner_base::output_character(ch);
}


void lua_cleaner::process_characters(char32_t const *begin, char32_t const *end)
{
	while (begin != end)
	{
		char32_t const ch(*begin++);
		switch (m_parse_state)
		{
		case parse_state::DEFAULT:
			process_default(ch);
			break;
		case parse_state::SHORT_COMMENT:
			process_short_comment(ch);
			break;
		case parse_state::LONG_COMMENT:
			process_long_comment(ch);
			break;
		case parse_state::STRING_CONSTANT:
			process_string_constant(ch);
			break;
		case parse_state::LONG_STRING_CONSTANT:
			process_long_string_constant(ch);
			break;
		}

		if (LINE_FEED == ch)
			++m_input_line;
	}
}


void lua_cleaner::input_complete()
{
	switch (m_parse_state)
	{
	case parse_state::LONG_COMMENT:
		throw std::runtime_error(util::string_format("unterminated long comment beginning on line %1$u", m_block_line));
	case parse_state::STRING_CONSTANT:
		throw std::runtime_error(util::string_format("unterminated string literal on line %1$u", m_input_line));
	case parse_state::LONG_STRING_CONSTANT:
		throw std::runtime_error(util::string_format("unterminated long string literal beginning on line %1$u", m_block_line));
	default:
		break;
	}
}


void lua_cleaner::process_default(char32_t ch)
{
	switch (ch)
	{
	case DOUBLE_QUOTE:
	case SINGLE_QUOTE:
		m_string_quote = ch;
		m_parse_state = parse_state::STRING_CONSTANT;
		break;
	case HYPHEN_MINUS:
		if (m_escape)
		{
			m_comment_start = true;
			m_parse_state = parse_state::SHORT_COMMENT;
		}
		break;
	default:
		break;
	}
	if (0 <= m_long_bracket_level)
	{
		switch (ch)
		{
		case EQUALS:
			++m_long_bracket_level;
			break;
		case LEFT_BRACKET:
			m_block_line = m_input_line;
			m_block_level = m_long_bracket_level;
			m_parse_state = parse_state::LONG_STRING_CONSTANT;
			[[fallthrough]];
		default:
			m_long_bracket_level = -1;
		}
	}
	else if (LEFT_BRACKET == ch)
	{
		m_long_bracket_level = 0;
	}
	m_escape = (HYPHEN_MINUS == ch) && !m_escape;
	output_character(ch);
}


void lua_cleaner::process_short_comment(char32_t ch)
{
	if (0 <= m_long_bracket_level)
	{
		switch (ch)
		{
		case EQUALS:
			++m_long_bracket_level;
			break;
		case LEFT_BRACKET:
			m_block_line = m_input_line;
			m_block_level = m_long_bracket_level;
			m_parse_state = parse_state::LONG_COMMENT;
			set_tab_limit();
			[[fallthrough]];
		default:
			m_long_bracket_level = -1;
		}
	}
	else if (m_comment_start && (LEFT_BRACKET == ch))
	{
		m_long_bracket_level = 0;
	}
	else if (LINE_FEED == ch)
	{
		m_parse_state = parse_state::DEFAULT;
	}
	m_comment_start = false;
	output_character(ch);
}


void lua_cleaner::process_long_comment(char32_t ch)
{
	if (0 <= m_long_bracket_level)
	{
		switch (ch)
		{
		case EQUALS:
			++m_long_bracket_level;
			break;
		case RIGHT_BRACKET:
			if (m_long_bracket_level == m_block_level)
			{
				m_parse_state = parse_state::DEFAULT;
				reset_tab_limit();
			}
			else
			{
				m_long_bracket_level = 0;
			}
			break;
		default:
			m_long_bracket_level = -1;
		}
	}
	else if (RIGHT_BRACKET == ch)
	{
		m_long_bracket_level = 0;
	}
	output_character(ch);
}


void lua_cleaner::process_string_constant(char32_t ch)
{
	switch (ch)
	{
	case HORIZONTAL_TAB:
		++m_tabs_escaped;
		if (!m_escape)
			output_character(BACKSLASH);
		output_character(char32_t(std::uint8_t('t')));
		break;
	case LINE_FEED:
		if (m_escape)
		{
			++m_newlines_escaped;
			output_character(char32_t(std::uint8_t('n')));
		}
		else
		{
			throw std::runtime_error(util::string_format("unterminated string literal on line %1$u", m_input_line));
		}
		break;
	case VERTICAL_TAB:
		++m_tabs_escaped;
		if (!m_escape)
			output_character(BACKSLASH);
		output_character(char32_t(std::uint8_t('v')));
		break;
	default:
		output_character(ch);
		if (!m_escape && (m_string_quote == ch))
			m_parse_state = parse_state::DEFAULT;
	}
	m_escape = (BACKSLASH == ch) && !m_escape;
}


void lua_cleaner::process_long_string_constant(char32_t ch)
{
	// this works because they're both closed by a matching long bracket
	process_long_comment(ch);
}



/***************************************************************************
    JSON DATA CLEANER CLASS
***************************************************************************/

class json_cleaner : public cleaner_base
{
public:
	template <typename OutputIt>
	json_cleaner(OutputIt &&output, newline newline_mode, unsigned tab_width);

	virtual bool affected() const override;
	virtual void summarise(std::ostream &os) const override;

protected:
	void output_character(char32_t ch);

private:
	enum class parse_state
	{
		DEFAULT,
		COMMENT,
		LINE_COMMENT,
		STRING_CONSTANT
	};

	virtual void process_characters(char32_t const *begin, char32_t const *end) override;
	virtual void input_complete() override;

	void process_default(char32_t ch);
	void process_comment(char32_t ch);
	void process_line_comment(char32_t ch);
	void process_text(char32_t ch);

	parse_state     m_parse_state   = parse_state::DEFAULT;
	std::uint64_t   m_input_line    = 1U;
	bool            m_escape        = false;
	std::uint64_t   m_comment_line  = 0U;

	std::uint64_t   m_tabs_escaped                  = 0U;
	std::uint64_t   m_newlines_escaped              = 0U;
	std::uint64_t   m_form_feeds_escaped            = 0U;
	std::uint64_t   m_line_separators_escaped       = 0U;
	std::uint64_t   m_paragraph_separators_escaped  = 0U;
	std::uint64_t   m_c0_control_escaped            = 0U;
	std::uint64_t   m_non_ascii                     = 0U;
};


template <typename OutputIt>
json_cleaner::json_cleaner(
		OutputIt &&output,
		newline newline_mode,
		unsigned tab_width)
	: cleaner_base(std::forward<OutputIt>(output), newline_mode, tab_width)
{
}


bool json_cleaner::affected() const
{
	return
			cleaner_base::affected() ||
			m_tabs_escaped ||
			m_newlines_escaped ||
			m_form_feeds_escaped ||
			m_line_separators_escaped ||
			m_paragraph_separators_escaped ||
			m_c0_control_escaped ||
			m_non_ascii;
}


void json_cleaner::summarise(std::ostream &os) const
{
	cleaner_base::summarise(os);
	if (m_tabs_escaped)
		util::stream_format(os, "%1$u tab(s) escaped\n", m_tabs_escaped);
	if (m_newlines_escaped)
		util::stream_format(os, "%1$u line feed(s) escaped\n", m_newlines_escaped);
	if (m_form_feeds_escaped)
		util::stream_format(os, "%1$u form feed(s) escaped\n", m_form_feeds_escaped);
	if (m_line_separators_escaped)
		util::stream_format(os, "%1$u line separator(s) escaped\n", m_line_separators_escaped);
	if (m_paragraph_separators_escaped)
		util::stream_format(os, "%1$u paragraph separator(s) escaped\n", m_paragraph_separators_escaped);
	if (m_c0_control_escaped)
		util::stream_format(os, "%1$u C0 control character(s) escaped\n", m_c0_control_escaped);
	if (m_non_ascii)
		util::stream_format(os, "%1$u non-ASCII character(s) replaced\n", m_non_ascii);
}


void json_cleaner::output_character(char32_t ch)
{
	switch (m_parse_state)
	{
	case parse_state::DEFAULT:
		if (BASIC_LATIN_LAST < ch)
		{
			++m_non_ascii;
			ch = QUESTION_MARK;
		}
		break;
	case parse_state::COMMENT:
	case parse_state::LINE_COMMENT:
	case parse_state::STRING_CONSTANT:
		break;
	}

	cleaner_base::output_character(ch);
}


void json_cleaner::process_characters(char32_t const *begin, char32_t const *end)
{
	while (begin != end)
	{
		char32_t const ch(*begin++);
		switch (m_parse_state)
		{
		case parse_state::DEFAULT:
			process_default(ch);
			break;
		case parse_state::COMMENT:
			process_comment(ch);
			break;
		case parse_state::LINE_COMMENT:
			process_line_comment(ch);
			break;
		case parse_state::STRING_CONSTANT:
			process_text(ch);
			break;
		}

		if (LINE_FEED == ch)
			++m_input_line;
	}
}


void json_cleaner::input_complete()
{
	switch (m_parse_state)
	{
	case parse_state::COMMENT:
		throw std::runtime_error(util::string_format("unterminated multi-line comment beginning on line %1$u", m_comment_line));
	case parse_state::STRING_CONSTANT:
		throw std::runtime_error(util::string_format("unterminated string literal on line %1$u", m_input_line));
	default:
		break;
	}
}


void json_cleaner::process_default(char32_t ch)
{
	switch (ch)
	{
	case DOUBLE_QUOTE:
		m_parse_state = parse_state::STRING_CONSTANT;
		break;
	case ASTERISK:
		if (m_escape)
		{
			m_parse_state = parse_state::COMMENT;
			m_comment_line = m_input_line;
			set_tab_limit();
		}
		break;
	case SLASH:
		if (m_escape)
			m_parse_state = parse_state::LINE_COMMENT;
		break;
	}
	m_escape = (SLASH == ch) && !m_escape;
	output_character(ch);
}


void json_cleaner::process_comment(char32_t ch)
{
	switch (ch)
	{
	case SLASH:
		if (m_escape)
		{
			m_parse_state = parse_state::DEFAULT;
			m_comment_line = 0U;
			output_character(ch);
			reset_tab_limit();
		}
		else
		{
			output_character(ch);
		}
		m_escape = false;
		break;
	default:
		m_escape = ASTERISK == ch;
		output_character(ch);
	}
}


void json_cleaner::process_line_comment(char32_t ch)
{
	switch (ch)
	{
	case LINE_FEED:
		m_parse_state = parse_state::DEFAULT;
		[[fallthrough]];
	default:
		output_character(ch);
	}
}


void json_cleaner::process_text(char32_t ch)
{
	switch (ch)
	{
	case HORIZONTAL_TAB:
		++m_tabs_escaped;
		if (!m_escape)
			output_character(BACKSLASH);
		output_character(char32_t(std::uint8_t('t')));
		break;
	case LINE_FEED:
		++m_newlines_escaped;
		if (!m_escape)
			output_character(BACKSLASH);
		output_character(char32_t(std::uint8_t('n')));
		break;
	case FORM_FEED:
		++m_form_feeds_escaped;
		if (!m_escape)
			output_character(BACKSLASH);
		output_character(char32_t(std::uint8_t('f')));
		break;
	case LINE_SEPARATOR:
		++m_line_separators_escaped;
		if (!m_escape)
			output_character(BACKSLASH);
		output_character(char32_t(std::uint8_t('u')));
		output_character(char32_t(std::uint8_t('2')));
		output_character(char32_t(std::uint8_t('0')));
		output_character(char32_t(std::uint8_t('2')));
		output_character(char32_t(std::uint8_t('8')));
		break;
	case PARAGRAPH_SEPARATOR:
		++m_paragraph_separators_escaped;
		if (!m_escape)
			output_character(BACKSLASH);
		output_character(char32_t(std::uint8_t('u')));
		output_character(char32_t(std::uint8_t('2')));
		output_character(char32_t(std::uint8_t('0')));
		output_character(char32_t(std::uint8_t('2')));
		output_character(char32_t(std::uint8_t('9')));
		break;
	default:
		if (C0_CONTROL_LAST >= ch)
		{
			++m_c0_control_escaped;
			if (!m_escape)
				output_character(BACKSLASH);
			output_character(char32_t(std::uint8_t('u')));
			output_character(char32_t(std::uint8_t('0')));
			output_character(char32_t(std::uint8_t('0')));
			output_character(char32_t(std::uint8_t('0') + (ch / 0x10U)));
			output_character(char32_t(std::uint8_t('0') + (ch % 0x10U)));
		}
		else
		{
			output_character(ch);
			if (!m_escape && (DOUBLE_QUOTE == ch))
				m_parse_state = parse_state::DEFAULT;
		}
	}
	m_escape = (BACKSLASH == ch) && !m_escape;
}



/***************************************************************************
    XML DATA CLEANER CLASS
***************************************************************************/

class xml_cleaner : public cleaner_base
{
public:
	template <typename OutputIt>
	xml_cleaner(OutputIt &&output, newline newline_mode, unsigned tab_width);

private:
	static constexpr char32_t EXCLAMATION           = 0x0000'0021U;
	static constexpr char32_t LEFT_ANGLE_BRACKET    = 0x0000'003cU;
	static constexpr char32_t RIGHT_ANGLE_BRACKET   = 0x0000'003eU;

	enum class parse_state
	{
		DEFAULT,
		COMMENT
	};

	virtual void process_characters(char32_t const *begin, char32_t const *end) override;
	virtual void input_complete() override;

	void process_default(char32_t ch);
	void process_comment(char32_t ch);

	parse_state     m_parse_state   = parse_state::DEFAULT;
	std::uint64_t   m_input_line    = 1U;
	unsigned        m_escape        = 0U;
	std::uint64_t   m_comment_line  = 0U;
};


template <typename OutputIt>
xml_cleaner::xml_cleaner(
		OutputIt &&output,
		newline newline_mode,
		unsigned tab_width)
	: cleaner_base(std::forward<OutputIt>(output), newline_mode, tab_width)
{
}


void xml_cleaner::process_characters(char32_t const *begin, char32_t const *end)
{
	while (begin != end)
	{
		char32_t const ch(*begin++);
		switch (m_parse_state)
		{
		case parse_state::DEFAULT:
			process_default(ch);
			break;
		case parse_state::COMMENT:
			process_comment(ch);
			break;
		}

		if (LINE_FEED == ch)
			++m_input_line;
	}
}


void xml_cleaner::input_complete()
{
	if (parse_state::COMMENT == m_parse_state)
		throw std::runtime_error(util::string_format("unterminated comment beginning on line %1$u", m_comment_line));
}


void xml_cleaner::process_default(char32_t ch)
{
	assert(4U > m_escape);

	switch (m_escape)
	{
	case 0U:
		m_escape = (LEFT_ANGLE_BRACKET == ch) ? (m_escape + 1U) : 0U;
		break;
	case 1U:
		m_escape = (EXCLAMATION == ch) ? (m_escape + 1U) : 0U;
		break;
	case 2U:
	case 3U:
		m_escape = (HYPHEN_MINUS == ch) ? (m_escape + 1U) : 0U;
		break;
	}
	output_character(ch);

	if (4U == m_escape)
	{
		m_parse_state = parse_state::COMMENT;
		m_escape = 0U;
		m_comment_line = m_input_line;
		set_tab_limit();
	}
}


void xml_cleaner::process_comment(char32_t ch)
{
	assert(3U > m_escape);

	switch (m_escape)
	{
	case 0U:
	case 1U:
		m_escape = (HYPHEN_MINUS == ch) ? (m_escape + 1U) : 0U;
		break;
	case 2U:
		m_escape = (RIGHT_ANGLE_BRACKET == ch) ? (m_escape + 1U) : (HYPHEN_MINUS == ch) ? m_escape : 0U;
		break;
	}
	output_character(ch);

	if (3U == m_escape)
	{
		m_parse_state = parse_state::DEFAULT;
		m_escape = 0U;
		m_comment_line = 0U;
		reset_tab_limit();
	}
}



/***************************************************************************
    UTILITY FUNCTIONS
***************************************************************************/

bool is_c_source_extension(char const *ext)
{
	return
			!core_stricmp(ext, ".c") ||
			!core_stricmp(ext, ".h") ||
			!core_stricmp(ext, ".cpp") ||
			!core_stricmp(ext, ".hpp") ||
			!core_stricmp(ext, ".ipp") ||
			!core_stricmp(ext, ".cxx") ||
			!core_stricmp(ext, ".hxx") ||
			!core_stricmp(ext, ".ixx") ||
			!core_stricmp(ext, ".lst");
}


bool is_lua_source_extension(char const *ext)
{
	return
			!core_stricmp(ext, ".lua");
}


bool is_json_extension(char const *ext)
{
	return
			!core_stricmp(ext, ".json");
}


bool is_xml_extension(char const *ext)
{
	return
			!core_stricmp(ext, ".hsi") ||
			!core_stricmp(ext, ".lay") ||
			!core_stricmp(ext, ".xml") ||
			!core_stricmp(ext, ".xslt");
}

} // anonymous namespace



/***************************************************************************
    MAIN
***************************************************************************/

int main(int argc, char *argv[])
{
	bool                    keep_backup(false);
	bool                    dry_run(false);
#if defined(_WIN32)
	cleaner_base::newline   newline_mode(cleaner_base::newline::DOS);
#else
	cleaner_base::newline   newline_mode(cleaner_base::newline::UNIX);
#endif
	for (bool arg_found = true; arg_found && (argc > 1); )
	{
		if (!std::strcmp(argv[1], "-b"))
			keep_backup = true;
		else if (!std::strcmp(argv[1], "-d"))
			dry_run = true;
		else if (!std::strcmp(argv[1], "-m"))
			newline_mode = cleaner_base::newline::MACINTOSH;
		else if (!std::strcmp(argv[1], "-u"))
			newline_mode = cleaner_base::newline::UNIX;
		else if (!std::strcmp(argv[1], "-w"))
			newline_mode = cleaner_base::newline::DOS;
		else
			arg_found = false;

		if (arg_found)
		{
			argc--;
			argv++;
		}
	}

	if (argc < 2)
	{
		printf("Usage: srcclean [-b] [-d] [-m] [-u] [-w] <file>...\n");
		return 0;
	}

	bool                affected(false);
	unsigned            failures(0U);
	char                original[1024];
	std::vector<char>   output;
	output.reserve(32 * 1024 * 1024);
	for (int i = 1; i < argc; ++i)
	{
		// open the file
		util::core_file::ptr infile;
		std::error_condition const err(util::core_file::open(argv[i], OPEN_FLAG_READ, infile));
		if (err)
		{
			if (affected)
				std::cerr << std::endl;
			affected = true;
			util::stream_format(std::cerr, "Can't open %1$s (%2$s)\n", argv[i], err.message());
			++failures;
			continue;
		}

		try
		{
			// instantiate appropriate cleaner implementation
			char const *const ext(std::strrchr(argv[i], '.'));
			bool const is_c_file(ext && is_c_source_extension(ext));
			bool const is_lua_file(ext && is_lua_source_extension(ext));
			bool const is_json_file(ext && is_json_extension(ext));
			bool const is_xml_file(ext && is_xml_extension(ext));
			std::unique_ptr<cleaner_base> cleaner;
			if (is_c_file)
				cleaner = std::make_unique<cpp_cleaner>(std::back_inserter(output), newline_mode, 4U);
			else if (is_lua_file)
				cleaner = std::make_unique<lua_cleaner>(std::back_inserter(output), newline_mode, 4U);
			else if (is_json_file)
				cleaner = std::make_unique<json_cleaner>(std::back_inserter(output), newline_mode, 4U);
			else if (is_xml_file)
				cleaner = std::make_unique<xml_cleaner>(std::back_inserter(output), newline_mode, 4U);
			else
				cleaner = std::make_unique<text_cleaner>(std::back_inserter(output), newline_mode, 4U);

			// read/process in chunks
			output.clear();
			std::uint64_t remaining(infile->size());
			std::uint32_t block;
			while (remaining && (0U != (block = infile->read(original, (std::min)(std::uint64_t(sizeof(original)), remaining)))))
			{
				remaining -= block;
				cleaner->process(original, original + block);
			}
			if (remaining)
			{
				if (affected)
					std::cerr << std::endl;
				affected = true;
				util::stream_format(std::cerr, "Can't read %1$s\n", argv[i]);
				++failures;
				continue;
			}
			cleaner->finalise();
			infile.reset();
			if (cleaner->affected())
			{
				// print report
				if (affected)
					std::cerr << std::endl;
				affected = true;
				util::stream_format(std::cerr, "Cleaned up %1$s:\n", argv[i]);
				cleaner->summarise(std::cerr);
				cleaner.reset();

				// replace the file if it isn't a dry run
				if (!dry_run)
				{
					using namespace std::string_literals;
					std::string const backup(argv[i] + ".orig"s);
					std::remove(backup.c_str());
					if (std::rename(argv[i], backup.c_str()))
					{
						util::stream_format(std::cerr, "Error moving %1$s to backup location\n", argv[i]);
						++failures;
					}
					else
					{
						std::ofstream outfile(argv[i], std::ios_base::binary | std::ios_base::out | std::ios_base::trunc);
						outfile.write(&output[0], output.size());
						outfile.flush();
						if (!outfile)
						{
							util::stream_format(std::cerr, "Error writing output to %1$s\n", argv[i]);
							++failures;
							outfile.close();
							if (std::rename(backup.c_str(), argv[i]))
								util::stream_format(std::cerr, "Error restoring backup of %1$s\n", argv[i]);
						}
						else if (!keep_backup)
						{
							if (std::remove(backup.c_str()))
							{
								util::stream_format(std::cerr, "Error removing backup of %1$s\n", argv[i]);
								++failures;
							}
						}
					}
				}
			}
		}
		catch (std::runtime_error const &ex)
		{
			// print error message and try the next file
			if (affected)
				std::cerr << std::endl;
			affected = true;
			util::stream_format(std::cerr, "Error cleaning %1$s: %2$s\n", argv[i], ex.what());
			++failures;
			continue;
		}
	}

	return failures ? 1 : 0;
}