WIP on palette with clusters for all textures

.gitignore

@@ -379,6 +379,7 @@ liberty/texconv*
 liberty/imgtool*
 liberty/coltool*
 liberty/streamheaderpack*
+liberty/merge-palettes*
 liberty/extract-sfx*
 liberty/pack-sfx*
 liberty/analyze-profile*

.vscode/settings.json

@@ -146,6 +146,9 @@
     "coroutine": "cpp",
     "future": "cpp",
     "latch": "cpp",
-    "syncstream": "cpp"
+    "syncstream": "cpp",
+    "mycommon.h": "c",
+    "file_tex.h": "c",
+    "optparse.h": "c"
   }
 }

liberty/Makefile

@@ -33,6 +33,9 @@ REPACK_STREAM_DECODED_DIR?=$(REPACK_DIR)/stream-decoded
 REPACK_STREAM_DEST_DIR=$(REPACK_GTA_DIR)/stream
 REPACK_CUTS_ORIG_DIR?=$(REPACK_DIR)/cuts-orig
 REPACK_CUTS_DC_DIR?=$(REPACK_DIR)/cuts-dc
+REPACK_MISC_TXD_EXTRACT_DIR?=$(REPACK_DIR)/misc-extract
+REPACK_TEXTURES_DIR?=$(REPACK_DIR)/textures
+REPACK_TEXTURE_CLUSTERS_DIR?=$(REPACK_DIR)/texture-clusters
 
 LIBS :=
 TEXCONV_FLAGS :=
@@ -304,12 +307,15 @@ coltool: ../src/tools/coltool.cpp
 streamheaderpack: ../src/tools/streamheaderpack.cpp
 	$(CXX) -std=c++17 -o $@ -g -O0 $<
 
+merge-palettes: ../src/tools/merge-palettes.cpp
+	$(CXX) -std=c++17 -o $@ -g -O0 $<
+
 -include $(DEPS)
 
 #### Repacking ####
 
-TXD_OPTS_fonts = 256 256
-TXD_OPTS_hud = 128 128
+TXD_OPTS_fonts = 512 512
+TXD_OPTS_hud = 512 512
 TXD_OPTS_menu = 512 512
 TXD_OPTS_LOADSC0 = 512 512
 TXD_OPTS_LOADSC1 = 512 512
@@ -358,12 +364,15 @@ TXD_OPTS_frontend = 512 512 \
 	--include-tex assets/ps4_f.png ps4_f \
 	--include-tex assets/xbox_d.png xbox_d \
 	--include-tex assets/xbox_f.png xbox_f
+TXD_OPTS_generic = 512 512
+TXD_OPTS_MISC = 512 512
+TXD_OPTS_particle = 512 512
 
 DEFAULT_RES = 512
 
 PVR_ENCODER ?= PVRTEX
 TEXTURE_DOWNSAMPLE_TXD ?= NONE
-TEXTURE_DOWNSAMPLE_IMG ?= HALF
+TEXTURE_DOWNSAMPLE_IMG ?= NONE
 
 -include texlist.mk
 -include modlist.mk
@@ -391,6 +400,11 @@ STREAM_ADPCM_DC = $(addprefix $(REPACK_STREAM_DEST_DIR)/, $(STREAM_WAV:.wav=.APM
 IMG_TEXTURES_ORIG = $(addprefix $(REPACK_IMG_ORIG_DIR)/, $(IMG_TEXTURES))
 IMG_MODELS_ORIG = $(addprefix $(REPACK_IMG_ORIG_DIR)/, $(IMG_MODELS))
 
+ALL_TEXTURES_ORIG = $(addprefix $(GTA_DIR)/,$(MISC_TXD)) $(IMG_TEXTURES_ORIG)
+ALL_TEXTURES_EXTRACT_PATH = $(addprefix $(REPACK_MISC_TXD_EXTRACT_DIR)/,$(MISC_TXD)) $(IMG_TEXTURES_ORIG)
+ALL_TEXTURES_ORIG_EXTRACT_txd = $(ALL_TEXTURES_EXTRACT_PATH:.txd=.extract)
+ALL_TEXTURES_ORIG_EXTRACT = $(ALL_TEXTURES_ORIG_EXTRACT_txd:.TXD=.extract)
+
 CUTS_IFP_ORIG = $(addprefix $(REPACK_CUTS_ORIG_DIR)/, $(CUTS_IFP))
 CUTS_MISC_ORIG = $(addprefix $(REPACK_CUTS_ORIG_DIR)/, $(CUTS_MISC))
 
@@ -401,12 +415,85 @@ SFX_REPACK_DC = $(SFX_REPACK_DC_WAV:.wav=.pcm)
 STREAM_MP3_DECODED = $(addprefix $(REPACK_STREAM_DECODED_DIR)/, $(STREAM_MP3:.mp3=.wav))
 STREAM_WAV_DECODED = $(addprefix $(REPACK_STREAM_DECODED_DIR)/, $(STREAM_WAV))
 
-.PRECIOUS: $(SFX_ORIG) $(SFX_REPACK_DC) $(STREAM_MP3_DECODED) $(STREAM_WAV_DECODED)
+.PRECIOUS: $(SFX_ORIG) $(SFX_REPACK_DC) $(STREAM_MP3_DECODED) $(STREAM_WAV_DECODED) $(ALL_TEXTURES_ORIG_EXTRACT)
 
-$(REPACK_DIR)/repacked: $(REPACK_GTA_DIR)/models/gta3.img $(REPACK_GTA_DIR)/models/gta3.dir $(LOOSE_FILES_DC) $(STREAM_ADPCM_DC) $(SFX_DC_RAW) $(SFX_DC_DSC) streamheaderpack
+# extracting textures from .txd
+
+# loose, gamefiles
+$(REPACK_MISC_TXD_EXTRACT_DIR)/%.extract: $(GTA_GAMEFILES_LOOSE_DIR)/%.TXD texconv
+	@rm -rf $(basename $@).contents
+	@mkdir -p $(basename $@).contents
+	@mkdir -p $(REPACK_TEXTURES_DIR)
+	./texconv $< $(basename $@) $(TXD_OPTS_$(notdir $*)) -e extract
+	@touch $@
+$(REPACK_MISC_TXD_EXTRACT_DIR)/%.extract: $(GTA_GAMEFILES_LOOSE_DIR)/%.txd texconv
+	@rm -rf $(basename $@).contents
+	@mkdir -p $(basename $@).contents
+	@mkdir -p $(REPACK_TEXTURES_DIR)
+	./texconv $< $(basename $@) $(TXD_OPTS_$(notdir $*)) -e extract
+	@touch $@
+
+# loose, originals
+$(REPACK_MISC_TXD_EXTRACT_DIR)/%.extract: $(GTA_DIR)/%.TXD texconv
+	@rm -rf $(basename $@).contents
+	@mkdir -p $(basename $@).contents
+	@mkdir -p $(REPACK_TEXTURES_DIR)
+	./texconv $< $(basename $@) $(TXD_OPTS_$(notdir $*)) -e extract
+	@touch $@
+$(REPACK_MISC_TXD_EXTRACT_DIR)/%.extract: $(GTA_DIR)/%.txd texconv
+	@rm -rf $(basename $@).contents
+	@mkdir -p $(basename $@).contents
+	@mkdir -p $(REPACK_TEXTURES_DIR)
+	./texconv $< $(basename $@) $(TXD_OPTS_$(notdir $*)) -e extract
+	@touch $@
+
+# img files
+$(REPACK_IMG_ORIG_DIR)/%.extract: $(REPACK_IMG_ORIG_DIR)/%.TXD texconv
+	@rm -rf $(basename $@).contents
+	@mkdir -p $(basename $@).contents
+	@mkdir -p $(REPACK_TEXTURES_DIR)
+	./texconv $< $(basename $@) 1024 1024 -e extract
+	@touch $@
+
+$(REPACK_IMG_ORIG_DIR)/%.extract: $(REPACK_IMG_ORIG_DIR)/%.txd texconv
+	@rm -rf $(basename $@).contents
+	@mkdir -p $(basename $@).contents
+	@mkdir -p $(REPACK_TEXTURES_DIR)
+	./texconv $< $(basename $@) 1024 1024 -e extract
+	@touch $@
+
+
+$(REPACK_DIR)/extracted: $(ALL_TEXTURES_ORIG_EXTRACT)
+	echo && echo && echo "Extracted TXDs" && echo && echo
+	@touch $@
+
+# Clustering them
+
+$(REPACK_DIR)/clustered: $(REPACK_DIR)/extracted ../src/tools/pal-clusters.py
+	echo "Creating clusters..."
+	python3 ../src/tools/pal-clusters.py $(REPACK_TEXTURES_DIR) $(REPACK_TEXTURE_CLUSTERS_DIR)
+	echo && echo && echo "Created texture clusters" && echo && echo
+	@touch $@
+
+# Palette creation
+
+CLUSTERS := $(shell seq 0 63)
+PAL_FILES := $(foreach cluster,$(CLUSTERS),$(REPACK_TEXTURE_CLUSTERS_DIR)/$(cluster).pal)
+
+$(REPACK_TEXTURE_CLUSTERS_DIR)/%.pal: $(REPACK_DIR)/clustered | pvrtex
+	../vendor/pvrtex/pvrtex $$(ls $(REPACK_TEXTURE_CLUSTERS_DIR)/cluster_$*/*.tga | sed 's/^/-i /') -f PAL4BPP -o $@
+
+$(REPACK_GTA_DIR)/dc-palette: $(REPACK_DIR)/clustered $(PAL_FILES) merge-palettes
+	./merge-palettes $(REPACK_TEXTURE_CLUSTERS_DIR) $(REPACK_GTA_DIR)/dc-palette
+	echo && echo && echo "Created cluster palettes" && echo && echo
+
+
+
+$(REPACK_DIR)/repacked: $(REPACK_GTA_DIR)/models/gta3.img $(REPACK_GTA_DIR)/models/gta3.dir $(LOOSE_FILES_DC) $(STREAM_ADPCM_DC) $(SFX_DC_RAW) $(SFX_DC_DSC) streamheaderpack $(REPACK_GTA_DIR)/dc-palette
 	mkdir -p $(@D)
 # $(REPACK_GTA_DIR) needed as first argument as paths in the game prefix with stream/
 	./streamheaderpack liberty $(REPACK_GTA_DIR) $(REPACK_STREAM_DEST_DIR)/hdr.bin
+	cp $(REPACK_GTA_DIR)/dc-palette $(REPACK_GTA_DIR)/dc-palette.pal
 	@git archive --format zip --output "$(REPACK_GTA_DIR)/DCA3-$(GIT_VERSION).zip" HEAD
 	@touch $@
 	@echo && echo && echo "*** Repack Completed Successfully ($(PROJECT_NAME)) ***" && echo && echo
@@ -445,18 +532,18 @@ $(REPACK_IMG_DC_DIR)/%.DFF: $(REPACK_IMG_ORIG_DIR)/%.DFF texconv
 	./texconv $< $@
 
 # first try the mods img directory. NB, the textures are not resized here, unlike normal .img textures
-$(REPACK_IMG_DC_DIR)/%.txd: $(GTA_MOD_IMG_DIR)/%.txd texconv
+$(REPACK_IMG_DC_DIR)/%.txd: $(GTA_MOD_IMG_DIR)/%.txd texconv $(REPACK_GTA_DIR)/dc-palette
 	@mkdir -p $(@D)
 	./texconv $< $@ 1024 1024 -e $(PVR_ENCODER) -d NONE
-$(REPACK_IMG_DC_DIR)/%.TXD: $(GTA_MOD_IMG_DIR)/%.TXD texconv
+$(REPACK_IMG_DC_DIR)/%.TXD: $(GTA_MOD_IMG_DIR)/%.TXD texconv $(REPACK_GTA_DIR)/dc-palette
 	@mkdir -p $(@D)
 	./texconv $< $@ 1024 1024 -e $(PVR_ENCODER) -d NONE
 
 # if not, the extracted img directory
-$(REPACK_IMG_DC_DIR)/%.txd: $(REPACK_IMG_ORIG_DIR)/%.txd texconv
+$(REPACK_IMG_DC_DIR)/%.txd: $(REPACK_IMG_ORIG_DIR)/%.txd texconv $(REPACK_GTA_DIR)/dc-palette
 	@mkdir -p $(@D)
 	./texconv $< $@ $(DEFAULT_RES) $(DEFAULT_RES) -e $(PVR_ENCODER) -d $(TEXTURE_DOWNSAMPLE_IMG)
-$(REPACK_IMG_DC_DIR)/%.TXD: $(REPACK_IMG_ORIG_DIR)/%.TXD texconv
+$(REPACK_IMG_DC_DIR)/%.TXD: $(REPACK_IMG_ORIG_DIR)/%.TXD texconv $(REPACK_GTA_DIR)/dc-palette
 	@mkdir -p $(@D)
 	./texconv $< $@ $(DEFAULT_RES) $(DEFAULT_RES) -e $(PVR_ENCODER) -d $(TEXTURE_DOWNSAMPLE_IMG)
 
@@ -502,26 +589,26 @@ $(REPACK_GTA_DIR)/%.DFF: $(GTA_DIR)/%.DFF texconv
 
 # first try the mods loose directory
 # Note the mods loose directory is not resized, unlike the normal .txd textures
-$(REPACK_GTA_DIR)/%.txd: $(GTA_MOD_LOOSE_DIR)/%.txd texconv
+$(REPACK_GTA_DIR)/%.txd: $(GTA_MOD_LOOSE_DIR)/%.txd texconv $(REPACK_GTA_DIR)/dc-palette
 	@mkdir -p $(@D)
 	./texconv $< $@ 1024 1024 -e $(PVR_ENCODER) -d NONE
-$(REPACK_GTA_DIR)/%.TXD: $(GTA_MOD_LOOSE_DIR)/%.TXD texconv
+$(REPACK_GTA_DIR)/%.TXD: $(GTA_MOD_LOOSE_DIR)/%.TXD texconv $(REPACK_GTA_DIR)/dc-palette
 	@mkdir -p $(@D)
 	./texconv $< $@ 1024 1024 -e $(PVR_ENCODER) -d NONE
 
 # then the gamefiles directory
-$(REPACK_GTA_DIR)/%.txd: $(GTA_GAMEFILES_LOOSE_DIR)/%.txd texconv
+$(REPACK_GTA_DIR)/%.txd: $(GTA_GAMEFILES_LOOSE_DIR)/%.txd texconv $(REPACK_GTA_DIR)/dc-palette
 	@mkdir -p $(@D)
 	./texconv $< $@ $(TXD_OPTS_$(notdir $*)) -e $(PVR_ENCODER) -d $(TEXTURE_DOWNSAMPLE_TXD)
-$(REPACK_GTA_DIR)/%.TXD: $(GTA_GAMEFILES_LOOSE_DIR)/%.TXD texconv
+$(REPACK_GTA_DIR)/%.TXD: $(GTA_GAMEFILES_LOOSE_DIR)/%.TXD texconv $(REPACK_GTA_DIR)/dc-palette
 	@mkdir -p $(@D)
 	./texconv $< $@ $(TXD_OPTS_$(notdir $*)) -e $(PVR_ENCODER) -d $(TEXTURE_DOWNSAMPLE_TXD)
 
 # if not, the original files
-$(REPACK_GTA_DIR)/%.txd: $(GTA_DIR)/%.txd texconv
+$(REPACK_GTA_DIR)/%.txd: $(GTA_DIR)/%.txd texconv $(REPACK_GTA_DIR)/dc-palette
 	@mkdir -p $(@D)
 	./texconv $< $@ $(TXD_OPTS_$(notdir $*)) -e $(PVR_ENCODER) -d $(TEXTURE_DOWNSAMPLE_TXD)
-$(REPACK_GTA_DIR)/%.TXD: $(GTA_DIR)/%.TXD texconv
+$(REPACK_GTA_DIR)/%.TXD: $(GTA_DIR)/%.TXD texconv $(REPACK_GTA_DIR)/dc-palette
 	@mkdir -p $(@D)
 	./texconv $< $@ $(TXD_OPTS_$(notdir $*)) -e $(PVR_ENCODER) -d $(TEXTURE_DOWNSAMPLE_TXD)
 

liberty/gta3files.mk

@@ -1,6 +1,46 @@
 # models/coll/peds.col # not actually used
+MISC_TXD = \
+  models/fonts.txd \
+  models/frontend.txd \
+  models/generic.txd \
+  models/hud.txd \
+  models/menu.txd \
+  models/MISC.TXD \
+  models/particle.txd \
+  txd/LOADSC0.TXD \
+  txd/LOADSC1.TXD \
+  txd/LOADSC10.TXD \
+  txd/LOADSC11.TXD \
+  txd/LOADSC12.TXD \
+  txd/LOADSC13.TXD \
+  txd/LOADSC14.TXD \
+  txd/LOADSC15.TXD \
+  txd/LOADSC16.TXD \
+  txd/LOADSC17.TXD \
+  txd/LOADSC18.TXD \
+  txd/LOADSC19.TXD \
+  txd/LOADSC2.TXD \
+  txd/LOADSC20.TXD \
+  txd/LOADSC21.TXD \
+  txd/LOADSC22.TXD \
+  txd/LOADSC23.TXD \
+  txd/LOADSC24.TXD \
+  txd/LOADSC25.TXD \
+  txd/LOADSC3.TXD \
+  txd/LOADSC4.TXD \
+  txd/LOADSC5.TXD \
+  txd/LOADSC6.TXD \
+  txd/LOADSC7.TXD \
+  txd/LOADSC8.TXD \
+  txd/LOADSC9.TXD \
+  txd/mainsc1.txd  \
+  txd/mainsc2.txd  \
+  txd/NEWS.TXD \
+  txd/SPLASH1.TXD \
+  txd/SPLASH2.TXD \
+  txd/SPLASH3.TXD
 
-MISC_FILES = \
+MISC_FILES = $(MISC_TXD) \
   anim/cuts.dir \
   anim/cuts.img \
   anim/gta3.ini \
@@ -136,44 +176,4 @@ MISC_FILES = \
   TEXT/italian.gxt \
   TEXT/spanish.gxt \
   \
-  models/fonts.txd \
-  models/frontend.txd \
-  models/generic.txd \
-  models/hud.txd \
-  models/menu.txd \
-  models/MISC.TXD \
-  models/particle.txd \
-  txd/LOADSC0.TXD \
-  txd/LOADSC1.TXD \
-  txd/LOADSC10.TXD \
-  txd/LOADSC11.TXD \
-  txd/LOADSC12.TXD \
-  txd/LOADSC13.TXD \
-  txd/LOADSC14.TXD \
-  txd/LOADSC15.TXD \
-  txd/LOADSC16.TXD \
-  txd/LOADSC17.TXD \
-  txd/LOADSC18.TXD \
-  txd/LOADSC19.TXD \
-  txd/LOADSC2.TXD \
-  txd/LOADSC20.TXD \
-  txd/LOADSC21.TXD \
-  txd/LOADSC22.TXD \
-  txd/LOADSC23.TXD \
-  txd/LOADSC24.TXD \
-  txd/LOADSC25.TXD \
-  txd/LOADSC3.TXD \
-  txd/LOADSC4.TXD \
-  txd/LOADSC5.TXD \
-  txd/LOADSC6.TXD \
-  txd/LOADSC7.TXD \
-  txd/LOADSC8.TXD \
-  txd/LOADSC9.TXD \
-  txd/mainsc1.txd  \
-  txd/mainsc2.txd  \
-  txd/NEWS.TXD \
-  txd/SPLASH1.TXD \
-  txd/SPLASH2.TXD \
-  txd/SPLASH3.TXD \
-  \
   audio/sfx.SDT

src/liberty/animation/AnimManager.cpp

@@ -11,6 +11,8 @@
 #include "AnimBlendAssocGroup.h"
 #include "AnimManager.h"
 
+void* re3StreamingAlloc(size_t size);
+
 CAnimBlock CAnimManager::ms_aAnimBlocks[NUMANIMBLOCKS];
 CAnimBlendHierarchy CAnimManager::ms_aAnimations[NUMANIMATIONS];
 int32 CAnimManager::ms_numAnimBlocks;
@@ -837,7 +839,7 @@ CAnimManager::LoadAnimFile(int fd, bool compress)
 			uint16_t flags;
 			CFileMgr::Read(fd, (char*)&flags, sizeof(flags));
 
-			seq->keyFrames = RwMalloc(dataSize);
+			seq->keyFrames = re3StreamingAlloc(dataSize);
 			assert(seq->keyFrames);
 			CFileMgr::Read(fd, (char*)seq->keyFrames, dataSize - sizeof(flags));
 			seq->type = flags;

src/liberty/audio/sampman_dc.cpp

@@ -175,6 +175,12 @@ file_t fdPedSfx;
 volatile uint32 nPedSfxReqReadId = 1;
 volatile uint32 nPedSfxReqNextId = 1;
 
+// this is very wasteful and temporary
+#define BANK_STAGE_SIZE 16 * 2048
+static  uint8_t stagingBufferBank[BANK_STAGE_SIZE] __attribute__((aligned(32)));
+std::mutex stagingBufferMtx;
+
+
 static int32 DCStreamedLength[TOTAL_STREAMED_SOUNDS];
 
 struct WavHeader {
@@ -568,26 +574,29 @@ cSampleManager::LoadSampleBank(uint8 nBank)
 		// TODO: Split per-bank sfx file
 		int fd = fs_open(SampleBankDataFilename, O_RDONLY);
 		assert(fd >= 0);
-		// this is very wasteful and temporary
-		void* stagingBuffer = memalign(32, 32 * 2048);
-		assert(stagingBuffer != 0);
+		
+		
+		{
+			std::lock_guard lk(stagingBufferMtx); // for stagingBufferBank
+		
+			void* stagingBuffer = stagingBufferBank;
 
-		// Ideally, we'd suspend the CdStream thingy here or read via that instead
-		uintptr_t loadOffset = bank.base;
-		fs_seek(fd, fileStart, SEEK_SET);
+			// Ideally, we'd suspend the CdStream thingy here or read via that instead
+			uintptr_t loadOffset = bank.base;
+			fs_seek(fd, fileStart, SEEK_SET);
 
-		while (fileSize > 0) {
-			size_t readSize = fileSize > 32 * 2048 ? 32 * 2048 : fileSize;
-			int rs = fs_read(fd, stagingBuffer, readSize);
-			debugf("Read %d bytes, expected %d\n", rs, readSize);
-			assert(rs == readSize);
-			spu_memload(loadOffset, stagingBuffer, readSize);
-			loadOffset += readSize;
-			fileSize -= readSize;
-			debugf("Loaded %d bytes, %d remaining\n", readSize, fileSize);
+			while (fileSize > 0) {
+				size_t readSize = fileSize > sizeof(stagingBufferBank) ? sizeof(stagingBufferBank) : fileSize;
+				int rs = fs_read(fd, stagingBuffer, readSize);
+				debugf("Read %d bytes, expected %d\n", rs, readSize);
+				assert(rs == readSize);
+				spu_memload(loadOffset, stagingBuffer, readSize);
+				loadOffset += readSize;
+				fileSize -= readSize;
+				debugf("Loaded %d bytes, %d remaining\n", readSize, fileSize);
+			}
 		}
 		fs_close(fd);
-		free(stagingBuffer);
 		
 
 		for (int nSfx = BankStartOffset[nBank]; nSfx < BankStartOffset[nBank+1]; nSfx++) {
@@ -736,15 +745,19 @@ cSampleManager::LoadPedComment(uint32 nComment)
 		// TODO: When we can dma directly to AICA, we can use this instead
 		// fs_read(fdPedSfx, SPU_BASE_U8 + (uintptr_t)cmd->dest, cmd->size);
 
-		void* stagingBuffer = memalign(32, cmd->size);
-		assert(stagingBuffer != 0);
-		debugf("Allocated %d bytes at %p\n", cmd->size, stagingBuffer);
-		int rs = fs_read(fdPedSfx, stagingBuffer, cmd->size);
-		debugf("Read %d bytes, expected %d\n", rs, cmd->size);
-		assert(rs == cmd->size);
-
-		spu_memload((uintptr_t)cmd->dest, stagingBuffer, cmd->size);
-		free(stagingBuffer);
+		assert(cmd->size < sizeof(stagingBufferBank));
+		{
+			std::lock_guard lk(stagingBufferMtx); // for stagingBufferBank
+			void* stagingBuffer = stagingBufferBank;
+			assert(stagingBuffer != 0);
+			debugf("Allocated %d bytes at %p\n", cmd->size, stagingBuffer);
+			int rs = fs_read(fdPedSfx, stagingBuffer, cmd->size);
+			debugf("Read %d bytes, expected %d\n", rs, cmd->size);
+			assert(rs == cmd->size);
+	
+			spu_memload((uintptr_t)cmd->dest, stagingBuffer, cmd->size);
+		}
+		
 		nPedSfxReqReadId = nPedSfxReqReadId + 1;
 	});
 
@@ -1268,6 +1281,8 @@ cSampleManager::InitialiseSampleBanks(void)
 		assert(m_aSamples[nComment].nByteSize <= PED_BLOCKSIZE_ADPCM);
 	}
 
+	assert(PED_BLOCKSIZE_ADPCM <= BANK_STAGE_SIZE);
+
 	LoadSampleBank(SFX_BANK_0);
 	
 	return TRUE;

src/liberty/collision/ColModel.cpp

@@ -2,6 +2,9 @@
 #include "ColModel.h"
 #include "Game.h"
 #include "MemoryHeap.h"
+#include "Collision.h"
+
+void* re3StreamingAlloc(size_t size);
 
 CColModel::CColModel(void)
 {
@@ -22,12 +25,12 @@ CColModel::CColModel(void)
 CColModel::~CColModel(void)
 {
 	RemoveCollisionVolumes();
-	RemoveTrianglePlanes();
 }
 
 void
 CColModel::RemoveCollisionVolumes(void)
 {
+	CCollision::RemoveTrianglePlanes(this);
 	if(ownsCollisionVolumes){
 		RwFree(spheres);
 		RwFree(lines);
@@ -93,6 +96,8 @@ CColModel::operator=(const CColModel &other)
 	int i;
 	int numVerts;
 
+	CCollision::RemoveTrianglePlanes(this);
+
 	boundingSphere = other.boundingSphere;
 	boundingBox = other.boundingBox;
 
@@ -163,7 +168,7 @@ CColModel::operator=(const CColModel &other)
 		if(vertices)
 			RwFree(vertices);
 		if(numVerts){
-			vertices = (CompressedVector*)RwMalloc(numVerts*sizeof(CompressedVector));
+			vertices = (CompressedVector*)re3StreamingAlloc(numVerts*sizeof(CompressedVector));
 			for(i = 0; i < numVerts; i++)
 				vertices[i] = other.vertices[i];
 		}
@@ -173,7 +178,7 @@ CColModel::operator=(const CColModel &other)
 			numTriangles = other.numTriangles;
 			if(triangles)
 				RwFree(triangles);
-			triangles = (CColTriangle*)RwMalloc(numTriangles*sizeof(CColTriangle));
+			triangles = (CColTriangle*)re3StreamingAlloc(numTriangles*sizeof(CColTriangle));
 		}
 		for(i = 0; i < numTriangles; i++)
 			triangles[i] = other.triangles[i];

src/liberty/collision/Collision.cpp

@@ -2287,6 +2287,15 @@ CCollision::DistToLine(const CVector *l0, const CVector *l1, const CVector *poin
 	return (*point - closest).Magnitude();
 }
 
+void
+CCollision::RemoveTrianglePlanes(CColModel *model)
+{
+	if(model->trianglePlanes){
+		ms_colModelCache.Remove(model->GetLinkPtr());
+		model->RemoveTrianglePlanes();
+	}
+}
+
 void
 CCollision::CalculateTrianglePlanes(CColModel *model)
 {

src/liberty/collision/Collision.h

@@ -41,6 +41,7 @@ public:
 	static void DrawColModel(const CMatrix &mat, const CColModel &colModel);
 	static void DrawColModel_Coloured(const CMatrix &mat, const CColModel &colModel, int32 id);
 
+	static void RemoveTrianglePlanes(CColModel *model);
 	static void CalculateTrianglePlanes(CColModel *model);
 
 	// all these return true if there's a collision

src/liberty/core/FileLoader.cpp

@@ -28,6 +28,8 @@
 
 #include <kos/dbglog.h>
 
+void* re3StreamingAlloc(size_t size);
+
 char CFileLoader::ms_line[256];
 
 const char*
@@ -221,7 +223,7 @@ CFileLoader::LoadCollisionFile(const char *filename)
 
 		mi = CModelInfo::GetModelInfo(modelname, nil);
 		if(mi){
-			if(mi->GetColModel()){
+			if(mi->GetColModel() && mi->DoesOwnColModel()){
 				LoadCollisionModel(work_buff+24, *mi->GetColModel(), modelname);
 			}else{
 				CColModel *model = new CColModel;
@@ -255,6 +257,24 @@ CFileLoader::LoadCollisionModel(uint8 *buf, CColModel &model, char *modelname)
 	model.boundingBox.max.z = *(float*)(buf+36);
 	model.numSpheres = *(int16*)(buf+40);
 	buf += 44;
+	if (model.spheres) {
+		RwFree(model.spheres);
+	}
+	if (model.lines) {
+		RwFree(model.lines);
+	}
+	if (model.boxes) {
+		RwFree(model.boxes);
+	}
+	if (model.vertices) {
+		RwFree(model.vertices);
+	}
+	if (model.triangles) {
+		RwFree(model.triangles);
+	}
+	if (model.trianglePlanes) {
+		CCollision::RemoveTrianglePlanes(&model);
+	}
 	if(model.numSpheres > 0){
 		model.spheres = (CColSphere*)RwMalloc(model.numSpheres*sizeof(CColSphere));
 		REGISTER_MEMPTR(&model.spheres);
@@ -292,7 +312,7 @@ CFileLoader::LoadCollisionModel(uint8 *buf, CColModel &model, char *modelname)
 	int32 numVertices = *(int16*)buf;
 	buf += 4;
 	if(numVertices > 0){
-		model.vertices = (CompressedVector*)RwMalloc(numVertices*sizeof(CompressedVector));
+		model.vertices = (CompressedVector*)re3StreamingAlloc(numVertices*sizeof(CompressedVector));
 		REGISTER_MEMPTR(&model.vertices);
 		for(i = 0; i < numVertices; i++){
 			model.vertices[i].SetFixed(*(int16*)buf, *(int16*)(buf+2), *(int16*)(buf+4));
@@ -304,7 +324,7 @@ CFileLoader::LoadCollisionModel(uint8 *buf, CColModel &model, char *modelname)
 	model.numTriangles = *(int16*)buf;
 	buf += 4;
 	if(model.numTriangles > 0){
-		model.triangles = (CColTriangle*)RwMalloc(model.numTriangles*sizeof(CColTriangle));
+		model.triangles = (CColTriangle*)re3StreamingAlloc(model.numTriangles*sizeof(CColTriangle));
 		REGISTER_MEMPTR(&model.triangles);
 		for(i = 0; i < model.numTriangles; i++){
 			model.triangles[i].Set(model.vertices, *(uint16*)buf, *(uint16*)(buf+2), *(uint16*)(buf+4), buf[6], buf[7]);

src/liberty/core/Streaming.cpp

@@ -1170,6 +1170,24 @@ bool re3EmergencyRemoveModel() {
 	return usedmem != CStreaming::ms_memoryUsed;
 }
 
+void* re3StreamingAlloc(size_t size) {
+	auto rv = RwMalloc(size);
+
+	while (rv == nil) {
+		if (re3RemoveLeastUsedModel()) {
+			rv = RwMalloc(size);
+			continue;
+		}
+		if (re3EmergencyRemoveModel()) {
+			rv = RwMalloc(size);
+			continue;
+		}
+		return nil;
+	}
+
+	return rv;
+}
+
 bool
 CStreaming::RemoveLeastUsedModel(void)
 {

src/liberty/modelinfo/BaseModelInfo.cpp

@@ -38,6 +38,14 @@ CBaseModelInfo::DeleteCollisionModel(void)
 	}
 }
 
+void CBaseModelInfo::SetColModel(CColModel *col, bool owns) {
+	if (m_bOwnsColModel) {
+		delete m_colModel;
+	}
+	m_colModel = col;
+	m_bOwnsColModel = owns;
+}
+
 void
 CBaseModelInfo::AddRef(void)
 {

src/liberty/modelinfo/BaseModelInfo.h

@@ -56,8 +56,7 @@ public:
 	}
 	char *GetModelName(void) { return m_name; }
 	void SetModelName(const char *name) { strncpy(m_name, name, MAX_MODEL_NAME); }
-	void SetColModel(CColModel *col, bool owns = false){
-		m_colModel = col; m_bOwnsColModel = owns; }
+	void SetColModel(CColModel *col, bool owns = false);
 	CColModel *GetColModel(void) { return m_colModel; }
 	bool DoesOwnColModel(void) { return m_bOwnsColModel; }
 	void DeleteCollisionModel(void);

src/liberty/objects/CutsceneHead.cpp

@@ -197,6 +197,10 @@ CCutsceneHead::PlayAnimation(const char *animName)
 		RwStreamSkip(stream, offset*2048);
 		if(RwStreamFindChunk(stream, rwID_HANIMANIMATION, nil, nil)){
 			anim = RpHAnimAnimationStreamRead(stream);
+			if (hier->interpolator->currentAnim) {
+				RpHAnimAnimationDestroy(hier->interpolator->currentAnim);
+				hier->interpolator->currentAnim = nil;
+			}
 			RpHAnimHierarchySetCurrentAnim(hier, anim);
 		}
 

src/liberty/save/PCSave.cpp

@@ -17,6 +17,8 @@
 
 #include "vmu/vmu.h"
 
+void* re3StreamingAlloc(size_t size);
+
 const char* _psGetUserFilesFolder();
 
 C_PcSave PcSaveHelper;
@@ -93,16 +95,17 @@ uint32_t C_PcSave::PcClassLoadRoutine(int32 file, uint8 *data) {
 		return size;
 	} else {
 		size &= ~0x80000000;
-		uint8* compressed = (uint8*)malloc(size);
+		uint8* compressed = (uint8*)re3StreamingAlloc(size);
+		assert(compressed);
 		err = CFileMgr::Read(file, (const char*)compressed, size) != size;
 		if (err || CFileMgr::GetErrorReadWrite(file)) {
-			free(compressed);
+			RwFree(compressed);
 			return 0;
 		}
 
 		lzo_uint decompressed_size = 0;
 		auto crv = lzo1x_decompress(compressed, size, data, &decompressed_size, NULL);
-		free(compressed);
+		RwFree(compressed);
 		if (crv != LZO_E_OK) {
 			return 0;
 		}
@@ -117,31 +120,37 @@ uint32_t C_PcSave::PcClassLoadRoutine(int32 file, uint8 *data) {
 bool
 C_PcSave::PcClassSaveRoutine(int32 file, uint8 *data, uint32 size)
 {
-	void* wrkmem = malloc(LZO1X_1_MEM_COMPRESS);
-	uint8* compressed = (uint8*)malloc(size*2);
+	void* wrkmem = re3StreamingAlloc(LZO1X_1_MEM_COMPRESS);
+	assert(wrkmem);
+	uint8* compressed = (uint8*)re3StreamingAlloc(size*2);
+	assert(compressed);
 	lzo_uint compressed_size;
 	int crv = lzo1x_1_compress(data, size, compressed, &compressed_size, wrkmem);
-	free(wrkmem);
+	RwFree(wrkmem);
+
+	if (crv == LZO_E_OK && compressed_size >= size) {
+		crv = LZO_E_NOT_COMPRESSIBLE;
+	}
 
 	if (crv == LZO_E_OK) {
 		uint32_t compressed_size32 = compressed_size | 0x80000000;
 		bool err = CFileMgr::Write(file, (const char*)&compressed_size32, sizeof(compressed_size32)) != sizeof(compressed_size32);
 		if (err || CFileMgr::GetErrorReadWrite(file)) {
-			free(compressed);
+			RwFree(compressed);
 			nErrorCode = SAVESTATUS_ERR_SAVE_WRITE;
 			strncpy(SaveFileNameJustSaved, ValidSaveName, sizeof(ValidSaveName) - 1);
 			return false;
 		}
 
 		err = CFileMgr::Write(file, (const char*)compressed, compressed_size) != compressed_size;
-		free(compressed);
+		RwFree(compressed);
 		if (err || CFileMgr::GetErrorReadWrite(file)) {
 			nErrorCode = SAVESTATUS_ERR_SAVE_WRITE;
 			strncpy(SaveFileNameJustSaved, ValidSaveName, sizeof(ValidSaveName) - 1);
 			return false;
 		}
 	} else if (crv == LZO_E_NOT_COMPRESSIBLE) {
-		free(compressed);
+		RwFree(compressed);
 		uint32_t compressed_size32 = size;
 		bool err = CFileMgr::Write(file, (const char*)&compressed_size32, sizeof(compressed_size32)) != sizeof(compressed_size32);
 		if (err || CFileMgr::GetErrorReadWrite(file)) {
@@ -156,7 +165,7 @@ C_PcSave::PcClassSaveRoutine(int32 file, uint8 *data, uint32 size)
 			return false;
 		}
 	} else {
-		free(compressed);
+		RwFree(compressed);
 		return false;
 	}
 

src/miami/animation/AnimManager.cpp

@@ -12,6 +12,8 @@
 #include "AnimManager.h"
 #include "Streaming.h"
 
+void* re3StreamingAlloc(size_t size);
+
 CAnimBlock CAnimManager::ms_aAnimBlocks[NUMANIMBLOCKS];
 CAnimBlendHierarchy CAnimManager::ms_aAnimations[NUMANIMATIONS];
 int32 CAnimManager::ms_numAnimBlocks;
@@ -1312,7 +1314,7 @@ CAnimManager::LoadAnimFile(RwStream *stream, bool compress, char (*uncompressedA
 			uint16_t flags;
 			RwStreamRead(stream, &flags, sizeof(flags));
 
-			seq->keyFrames = RwMalloc(dataSize);
+			seq->keyFrames = re3StreamingAlloc(dataSize);
 			assert(seq->keyFrames);
 			RwStreamRead(stream, seq->keyFrames, dataSize - sizeof(flags));
 			seq->type = flags;

src/miami/animation/CutsceneMgr.cpp

@@ -419,7 +419,8 @@ CCutsceneMgr::DeleteCutsceneData(void)
 		CBaseModelInfo *minfo = CModelInfo::GetModelInfo(i);
 		CColModel *colModel = minfo->GetColModel();
 		if (colModel != &CTempColModels::ms_colModelPed1) {
-			delete colModel;
+			// no need to delete anymore, SetColModel will do it (~skmp)
+			//delete colModel;
 			minfo->SetColModel(&CTempColModels::ms_colModelPed1);
 		}
 	}

src/miami/audio/sampman_dc.cpp

@@ -182,6 +182,12 @@ uintptr_t gPlayerTalkData = 0;
 uint32 gPlayerTalkReqId = 0;
 #endif
 
+// this is very wasteful and temporary
+#define BANK_STAGE_SIZE 16 * 2048
+static  uint8_t stagingBufferBank[BANK_STAGE_SIZE] __attribute__((aligned(32)));
+std::mutex stagingBufferMtx;
+
+
 static int32 DCStreamedLength[TOTAL_STREAMED_SOUNDS];
 
 struct WavHeader {
@@ -581,26 +587,29 @@ cSampleManager::LoadSampleBank(uint8 nBank)
 		// TODO: Split per-bank sfx file
 		int fd = fs_open(SampleBankDataFilename, O_RDONLY);
 		assert(fd >= 0);
-		// this is very wasteful and temporary
-		void* stagingBuffer = memalign(32, 8 * 2048);
-		assert(stagingBuffer != 0);
-
-		// Ideally, we'd suspend the CdStream thingy here or read via that instead
-		uintptr_t loadOffset = bank.base;
+		
 		fs_seek(fd, fileStart, SEEK_SET);
+		{ 
+			std::lock_guard lk(stagingBufferMtx); // for stagingBufferBank
+		
+			void* stagingBuffer = stagingBufferBank;
+			assert(stagingBuffer != 0);
 
-		while (fileSize > 0) {
-			size_t readSize = fileSize > 8 * 2048 ? 8 * 2048 : fileSize;
-			int rs = fs_read(fd, stagingBuffer, readSize);
-			debugf("Read %d bytes, expected %d\n", rs, readSize);
-			assert(rs == readSize);
-			spu_memload(loadOffset, stagingBuffer, readSize);
-			loadOffset += readSize;
-			fileSize -= readSize;
-			debugf("Loaded %d bytes, %d remaining\n", readSize, fileSize);
+			// Ideally, we'd suspend the CdStream thingy here or read via that instead
+			uintptr_t loadOffset = bank.base;
+
+			while (fileSize > 0) {
+				size_t readSize = fileSize > sizeof(stagingBufferBank) ? sizeof(stagingBufferBank) : fileSize;
+				int rs = fs_read(fd, stagingBuffer, readSize);
+				debugf("Read %d bytes, expected %d\n", rs, readSize);
+				assert(rs == readSize);
+				spu_memload(loadOffset, stagingBuffer, readSize);
+				loadOffset += readSize;
+				fileSize -= readSize;
+				debugf("Loaded %d bytes, %d remaining\n", readSize, fileSize);
+			}
 		}
 		fs_close(fd);
-		free(stagingBuffer);
 		
 
 		for (int nSfx = BankStartOffset[nBank]; nSfx < BankStartOffset[nBank+1]; nSfx++) {
@@ -693,15 +702,19 @@ cSampleManager::LoadMissionAudio(uint8 nSlot, uint32 nSample)
 		// TODO: When we can dma directly to AICA, we can use this instead
 		// fs_read(fdPedSfx, SPU_BASE_U8 + (uintptr_t)cmd->dest, cmd->size);
 
-		void* stagingBuffer = memalign(32, cmd->size);
-		assert(stagingBuffer != 0);
-		debugf("Allocated %d bytes at %p\n", cmd->size, stagingBuffer);
-		int rs = fs_read(fdPedSfx, stagingBuffer, cmd->size);
-		debugf("Read %d bytes, expected %d\n", rs, cmd->size);
-		assert(rs == cmd->size);
-
-		spu_memload((uintptr_t)cmd->dest, stagingBuffer, cmd->size);
-		free(stagingBuffer);
+		assert(cmd->size < sizeof(stagingBufferBank));
+		{
+			std::lock_guard lk(stagingBufferMtx); // for stagingBufferBank
+			void* stagingBuffer = stagingBufferBank;
+			assert(stagingBuffer != 0);
+			debugf("Allocated %d bytes at %p\n", cmd->size, stagingBuffer);
+			int rs = fs_read(fdPedSfx, stagingBuffer, cmd->size);
+			debugf("Read %d bytes, expected %d\n", rs, cmd->size);
+			assert(rs == cmd->size);
+	
+			spu_memload((uintptr_t)cmd->dest, stagingBuffer, cmd->size);
+		}
+		
 		nPedSfxReqReadId = nPedSfxReqReadId + 1;
 	});
 	
@@ -787,15 +800,19 @@ cSampleManager::LoadPedComment(uint32 nComment)
 		// TODO: When we can dma directly to AICA, we can use this instead
 		// fs_read(fdPedSfx, SPU_BASE_U8 + (uintptr_t)cmd->dest, cmd->size);
 
-		void* stagingBuffer = memalign(32, cmd->size);
-		assert(stagingBuffer != 0);
-		debugf("Allocated %d bytes at %p\n", cmd->size, stagingBuffer);
-		int rs = fs_read(fdPedSfx, stagingBuffer, cmd->size);
-		debugf("Read %d bytes, expected %d\n", rs, cmd->size);
-		assert(rs == cmd->size);
+		assert(cmd->size < sizeof(stagingBufferBank));
+		{
+			std::lock_guard lk(stagingBufferMtx); // for stagingBufferBank
+			void* stagingBuffer = stagingBufferBank;
+			assert(stagingBuffer != 0);
+			debugf("Allocated %d bytes at %p\n", cmd->size, stagingBuffer);
+			int rs = fs_read(fdPedSfx, stagingBuffer, cmd->size);
+			debugf("Read %d bytes, expected %d\n", rs, cmd->size);
+			assert(rs == cmd->size);
+
+			spu_memload((uintptr_t)cmd->dest, stagingBuffer, cmd->size);
+		}
 
-		spu_memload((uintptr_t)cmd->dest, stagingBuffer, cmd->size);
-		free(stagingBuffer);
 		nPedSfxReqReadId = nPedSfxReqReadId + 1;
 	});
 
@@ -1349,16 +1366,21 @@ cSampleManager::InitialiseSampleBanks(void)
 	for (uint32 nComment = SAMPLEBANK_PED_START; nComment <= SAMPLEBANK_PED_END; nComment++) {
 		pedBlocksizeMax = Max(pedBlocksizeMax, m_aSamples[nComment].nByteSize);
 	}
+	assert(pedBlocksizeMax <= BANK_STAGE_SIZE);
 	debugf("Max ped comment size: %d\n", pedBlocksizeMax);
 
 #ifdef FIX_BUGS
 
 	// Find biggest player comment
 	uint32 nMaxPlayerSize = 0;
-	for (uint32 i = PLAYER_COMMENTS_START; i <= PLAYER_COMMENTS_END; i++)
+	for (uint32 i = PLAYER_COMMENTS_START; i <= PLAYER_COMMENTS_END; i++) {
 		nMaxPlayerSize = Max(nMaxPlayerSize, m_aSamples[i].nByteSize);
+	}
 
 	debugf("Max player comment size: %d\n", nMaxPlayerSize);
+
+	assert(nMaxPlayerSize < sizeof(stagingBufferBank));
+
 	gPlayerTalkData = snd_mem_malloc(nMaxPlayerSize);
 	ASSERT(gPlayerTalkData != 0);
 

src/miami/collision/ColModel.cpp

@@ -5,6 +5,8 @@
 #include "MemoryHeap.h"
 #include "Pools.h"
 
+void* re3StreamingAlloc(size_t size);
+
 CColModel::CColModel(void)
 {
 	numSpheres = 0;
@@ -43,13 +45,13 @@ CColModel::operator delete(void *p, size_t) throw()
 void
 CColModel::RemoveCollisionVolumes(void)
 {
+	CCollision::RemoveTrianglePlanes(this);
 	if(ownsCollisionVolumes){
 		RwFree(spheres);
 		RwFree(lines);
 		RwFree(boxes);
 		RwFree(vertices);
 		RwFree(triangles);
-		CCollision::RemoveTrianglePlanes(this);
 	}
 	numSpheres = 0;
 	numLines = 0;
@@ -109,6 +111,8 @@ CColModel::operator=(const CColModel &other)
 	int i;
 	int numVerts;
 
+	CCollision::RemoveTrianglePlanes(this);
+
 	boundingSphere = other.boundingSphere;
 	boundingBox = other.boundingBox;
 
@@ -179,7 +183,7 @@ CColModel::operator=(const CColModel &other)
 		if(vertices)
 			RwFree(vertices);
 		if(numVerts){
-			vertices = (CompressedVector*)RwMalloc(numVerts*sizeof(CompressedVector));
+			vertices = (CompressedVector*)re3StreamingAlloc(numVerts*sizeof(CompressedVector));
 			for(i = 0; i < numVerts; i++)
 				vertices[i] = other.vertices[i];
 		}
@@ -189,7 +193,7 @@ CColModel::operator=(const CColModel &other)
 			numTriangles = other.numTriangles;
 			if(triangles)
 				RwFree(triangles);
-			triangles = (CColTriangle*)RwMalloc(numTriangles*sizeof(CColTriangle));
+			triangles = (CColTriangle*)re3StreamingAlloc(numTriangles*sizeof(CColTriangle));
 		}
 		for(i = 0; i < numTriangles; i++)
 			triangles[i] = other.triangles[i];

src/miami/core/FileLoader.cpp

@@ -30,6 +30,8 @@
 #include "ColStore.h"
 #include "Occlusion.h"
 
+void* re3StreamingAlloc(size_t size);
+
 char CFileLoader::ms_line[256];
 
 const char*
@@ -303,6 +305,24 @@ CFileLoader::LoadCollisionModel(uint8 *buf, CColModel &model, char *modelname)
 	model.boundingBox.max.z = *(float*)(buf+36);
 	model.numSpheres = *(int16*)(buf+40);
 	buf += 44;
+	if (model.spheres) {
+		RwFree(model.spheres);
+	}
+	if (model.lines) {
+		RwFree(model.lines);
+	}
+	if (model.boxes) {
+		RwFree(model.boxes);
+	}
+	if (model.vertices) {
+		RwFree(model.vertices);
+	}
+	if (model.triangles) {
+		RwFree(model.triangles);
+	}
+	if (model.trianglePlanes) {
+		CCollision::RemoveTrianglePlanes(&model);
+	}
 	if(model.numSpheres > 0){
 		model.spheres = (CColSphere*)RwMalloc(model.numSpheres*sizeof(CColSphere));
 		REGISTER_MEMPTR(&model.spheres);
@@ -360,7 +380,7 @@ CFileLoader::LoadCollisionModel(uint8 *buf, CColModel &model, char *modelname)
 	model.numTriangles = *(int16*)buf;
 	buf += 4;
 	if(model.numTriangles > 0){
-		model.triangles = (CColTriangle*)RwMalloc(model.numTriangles*sizeof(CColTriangle));
+		model.triangles = (CColTriangle*)re3StreamingAlloc(model.numTriangles*sizeof(CColTriangle));
 		REGISTER_MEMPTR(&model.triangles);
 		for(i = 0; i < model.numTriangles; i++){
 			model.triangles[i].Set(*(uint16*)buf, *(uint16*)(buf+2), *(uint16*)(buf+4), buf[6]);

src/miami/core/Streaming.cpp

@@ -1386,6 +1386,24 @@ bool re3EmergencyRemoveModel() {
 	return usedmem != CStreaming::ms_memoryUsed;
 }
 
+void* re3StreamingAlloc(size_t size) {
+	auto rv = RwMalloc(size);
+
+	while (rv == nil) {
+		if (re3RemoveLeastUsedModel()) {
+			rv = RwMalloc(size);
+			continue;
+		}
+		if (re3EmergencyRemoveModel()) {
+			rv = RwMalloc(size);
+			continue;
+		}
+		return nil;
+	}
+
+	return rv;
+}
+
 bool
 CStreaming::RemoveLeastUsedModel(uint32 excludeMask)
 {

src/miami/modelinfo/BaseModelInfo.cpp

@@ -40,6 +40,14 @@ CBaseModelInfo::DeleteCollisionModel(void)
 	}
 }
 
+void CBaseModelInfo::SetColModel(CColModel *col, bool owns) {
+	if (m_bOwnsColModel) {
+		delete m_colModel;
+	}
+	m_colModel = col;
+	m_bOwnsColModel = owns;
+}
+
 void
 CBaseModelInfo::AddRef(void)
 {

src/miami/modelinfo/BaseModelInfo.h

@@ -52,8 +52,7 @@ public:
 	bool IsClump(void) { return m_type == MITYPE_CLUMP || m_type == MITYPE_PED || m_type == MITYPE_VEHICLE;	}
 	char *GetModelName(void) { return m_name; }
 	void SetModelName(const char *name) { strncpy(m_name, name, MAX_MODEL_NAME); }
-	void SetColModel(CColModel *col, bool owns = false){
-		m_colModel = col; m_bOwnsColModel = owns; }
+	void SetColModel(CColModel *col, bool owns = false);
 	CColModel *GetColModel(void) { return m_colModel; }
 	bool DoesOwnColModel(void) { return m_bOwnsColModel; }
 	void DeleteCollisionModel(void);

src/miami/renderer/ShadowCamera.cpp

@@ -271,13 +271,13 @@ CShadowCamera::InvertRaster()
 	RwIm2DVertexSetIntRGBA     (&vx[1], 255, 255, 255, 255);
 
 	RwIm2DVertexSetScreenX     (&vx[2], crw);
-	RwIm2DVertexSetScreenY     (&vx[2], 0.0f);
+	RwIm2DVertexSetScreenY     (&vx[2], crh);
 	RwIm2DVertexSetScreenZ     (&vx[2], RwIm2DGetNearScreenZ());
 	RwIm2DVertexSetRecipCameraZ(&vx[2], recipZ);
 	RwIm2DVertexSetIntRGBA     (&vx[2], 255, 255, 255, 255);
 
 	RwIm2DVertexSetScreenX     (&vx[3], crw);
-	RwIm2DVertexSetScreenY     (&vx[3], crh);
+	RwIm2DVertexSetScreenY     (&vx[3], 0.0f);
 	RwIm2DVertexSetScreenZ     (&vx[3], RwIm2DGetNearScreenZ());
 	RwIm2DVertexSetRecipCameraZ(&vx[3], recipZ);
 	RwIm2DVertexSetIntRGBA     (&vx[3], 255, 255, 255, 255);
@@ -289,7 +289,7 @@ CShadowCamera::InvertRaster()
 	RwRenderStateSet(rwRENDERSTATESRCBLEND,          (void *)rwBLENDINVDESTCOLOR);
 	RwRenderStateSet(rwRENDERSTATEDESTBLEND,         (void *)rwBLENDZERO);
 
-	RwIm2DRenderPrimitive(rwPRIMTYPETRISTRIP, vx, 4);
+	RwIm2DRenderPrimitive(rwPRIMTYPETRIFAN, vx, 4);
 
 	RwRenderStateSet(rwRENDERSTATEZTESTENABLE,       (void *)TRUE);
 	RwRenderStateSet(rwRENDERSTATESRCBLEND,          (void *)rwBLENDSRCALPHA);

src/miami/rw/RwHelper.cpp

@@ -384,23 +384,23 @@ RwBool Im2DRenderQuad(RwReal x1, RwReal y1, RwReal x2, RwReal y2, RwReal z, RwRe
     RwIm2DVertexSetU(&vx[1], uvOffset, recipCamZ);
     RwIm2DVertexSetV(&vx[1], 1.0f + uvOffset, recipCamZ);
 
-    RwIm2DVertexSetScreenX(&vx[2], x2);
-    RwIm2DVertexSetScreenY(&vx[2], y1);
+	RwIm2DVertexSetScreenX(&vx[2], x2);
+    RwIm2DVertexSetScreenY(&vx[2], y2);
     RwIm2DVertexSetScreenZ(&vx[2], z);
     RwIm2DVertexSetIntRGBA(&vx[2], 255, 255, 255, 255);
     RwIm2DVertexSetRecipCameraZ(&vx[2], recipCamZ);
     RwIm2DVertexSetU(&vx[2], 1.0f + uvOffset, recipCamZ);
-    RwIm2DVertexSetV(&vx[2], uvOffset, recipCamZ);
-
+    RwIm2DVertexSetV(&vx[2], 1.0f + uvOffset, recipCamZ);
+	
     RwIm2DVertexSetScreenX(&vx[3], x2);
-    RwIm2DVertexSetScreenY(&vx[3], y2);
+    RwIm2DVertexSetScreenY(&vx[3], y1);
     RwIm2DVertexSetScreenZ(&vx[3], z);
     RwIm2DVertexSetIntRGBA(&vx[3], 255, 255, 255, 255);
     RwIm2DVertexSetRecipCameraZ(&vx[3], recipCamZ);
     RwIm2DVertexSetU(&vx[3], 1.0f + uvOffset, recipCamZ);
-    RwIm2DVertexSetV(&vx[3], 1.0f + uvOffset, recipCamZ);
+    RwIm2DVertexSetV(&vx[3], uvOffset, recipCamZ);
 
-    RwIm2DRenderPrimitive(rwPRIMTYPETRISTRIP, vx, 4);
+    RwIm2DRenderPrimitive(rwPRIMTYPETRIFAN, vx, 4);
 
     return TRUE;
 }

src/miami/save/PCSave.cpp

@@ -17,6 +17,8 @@
 
 #include "vmu/vmu.h"
 
+void* re3StreamingAlloc(size_t size);
+
 const char* _psGetUserFilesFolder();
 
 C_PcSave PcSaveHelper;
@@ -76,31 +78,37 @@ C_PcSave::SaveSlot(int32 slot)
 bool
 C_PcSave::PcClassSaveRoutine(int32 file, uint8 *data, uint32 size)
 {
-	void* wrkmem = malloc(LZO1X_1_MEM_COMPRESS);
-	uint8* compressed = (uint8*)malloc(size*2);
+	void* wrkmem = re3StreamingAlloc(LZO1X_1_MEM_COMPRESS);
+	assert(wrkmem);
+	uint8* compressed = (uint8*)re3StreamingAlloc(size*2);
+	assert(compressed);
 	lzo_uint compressed_size;
 	int crv = lzo1x_1_compress(data, size, compressed, &compressed_size, wrkmem);
-	free(wrkmem);
+	RwFree(wrkmem);
+
+	if (crv == LZO_E_OK && compressed_size >= size) {
+		crv = LZO_E_NOT_COMPRESSIBLE;
+	}
 
 	if (crv == LZO_E_OK) {
 		uint32_t compressed_size32 = compressed_size | 0x80000000;
 		bool err = CFileMgr::Write(file, (const char*)&compressed_size32, sizeof(compressed_size32)) != sizeof(compressed_size32);
 		if (err || CFileMgr::GetErrorReadWrite(file)) {
-			free(compressed);
+			RwFree(compressed);
 			nErrorCode = SAVESTATUS_ERR_SAVE_WRITE;
 			strncpy(SaveFileNameJustSaved, ValidSaveName, sizeof(ValidSaveName) - 1);
 			return false;
 		}
 
 		err = CFileMgr::Write(file, (const char*)compressed, compressed_size) != compressed_size;
-		free(compressed);
+		RwFree(compressed);
 		if (err || CFileMgr::GetErrorReadWrite(file)) {
 			nErrorCode = SAVESTATUS_ERR_SAVE_WRITE;
 			strncpy(SaveFileNameJustSaved, ValidSaveName, sizeof(ValidSaveName) - 1);
 			return false;
 		}
 	} else if (crv == LZO_E_NOT_COMPRESSIBLE) {
-		free(compressed);
+		RwFree(compressed);
 		uint32_t compressed_size32 = size;
 		bool err = CFileMgr::Write(file, (const char*)&compressed_size32, sizeof(compressed_size32)) != sizeof(compressed_size32);
 		if (err || CFileMgr::GetErrorReadWrite(file)) {
@@ -115,7 +123,7 @@ C_PcSave::PcClassSaveRoutine(int32 file, uint8 *data, uint32 size)
 			return false;
 		}
 	} else {
-		free(compressed);
+		RwFree(compressed);
 		return false;
 	}
 
@@ -153,16 +161,17 @@ uint32_t C_PcSave::PcClassLoadRoutine(int32 file, uint8 *data) {
 		return size;
 	} else {
 		size &= ~0x80000000;
-		uint8* compressed = (uint8*)malloc(size);
+		uint8* compressed = (uint8*)re3StreamingAlloc(size);
+		assert(compressed);
 		err = CFileMgr::Read(file, (const char*)compressed, size) != size;
 		if (err || CFileMgr::GetErrorReadWrite(file)) {
-			free(compressed);
+			RwFree(compressed);
 			return 0;
 		}
 
 		lzo_uint decompressed_size = 0;
 		auto crv = lzo1x_decompress(compressed, size, data, &decompressed_size, NULL);
-		free(compressed);
+		RwFree(compressed);
 		if (crv != LZO_E_OK) {
 			return 0;
 		}

src/tools/merge-palettes.cpp

@@ -0,0 +1,69 @@
+#include <iostream>
+#include <fstream>
+#include <vector>
+#include <string>
+#include <cstdint>
+
+int main(int argc, char* argv[]) {
+    if (argc != 3) {
+        std::cerr << "Usage: " << argv[0] << " <input_root_folder> <output_file>" << std::endl;
+        return 1;
+    }
+
+    std::string inputFolder = argv[1];
+    std::string outputFileName = argv[2];
+
+    // Open the output file for writing in binary mode.
+    std::ofstream outFile(outputFileName, std::ios::binary);
+    if (!outFile) {
+        std::cerr << "Error: Cannot open output file " << outputFileName << std::endl;
+        return 1;
+    }
+
+    // Process each file from 0.pal to 63.pal.
+    for (int i = 0; i < 64; ++i) {
+        std::string filePath = inputFolder + "/" + std::to_string(i) + ".pal";
+        std::ifstream inFile(filePath, std::ios::binary);
+        if (!inFile) {
+            std::cerr << "Error: Cannot open input file " << filePath << std::endl;
+            return 1;
+        }
+
+        // Read header (first 4 bytes). They should equal "DPAL".
+        char header[4];
+        inFile.read(header, 4);
+        if (inFile.gcount() != 4 || std::string(header, 4) != "DPAL") {
+            std::cerr << "Error: File " << filePath << " has an invalid header." << std::endl;
+            return 1;
+        }
+
+        // Read the next 4 bytes (number of palette entries).
+        uint32_t numEntries = 0;
+        inFile.read(reinterpret_cast<char*>(&numEntries), sizeof(numEntries));
+        if (inFile.gcount() != sizeof(numEntries)) {
+            std::cerr << "Error: Could not read palette entry count from " << filePath << std::endl;
+            return 1;
+        }
+        if (numEntries != 16) {
+            std::cerr << "Error: File " << filePath << " has " << numEntries 
+                      << " entries, expected 16." << std::endl;
+            return 1;
+        }
+
+        // Read the remaining bytes (the palette values).
+        std::vector<char> paletteData((std::istreambuf_iterator<char>(inFile)),
+                                      std::istreambuf_iterator<char>());
+        inFile.close();
+
+        // Write palette data to the output file.
+        outFile.write(paletteData.data(), paletteData.size());
+        if (!outFile) {
+            std::cerr << "Error: Writing to output file failed." << std::endl;
+            return 1;
+        }
+    }
+
+    outFile.close();
+    std::cout << "Successfully combined 64 .pal files into " << outputFileName << std::endl;
+    return 0;
+}

src/tools/pal-clusters.py

@@ -0,0 +1,128 @@
+import os
+import shutil
+import argparse
+import numpy as np
+from PIL import Image
+from sklearn.cluster import KMeans
+
+def compute_mse(original, quantized):
+    """
+    Compute Mean Squared Error between two images (numpy arrays).
+    """
+    if original.shape != quantized.shape:
+        raise ValueError("Images must have the same dimensions")
+    mse = np.mean((original - quantized) ** 2)
+    return mse
+
+def extract_palette_feature(quantized_image, num_colors=16):
+    """
+    Extract a flattened feature vector from the quantized image palette.
+    Uses the getcolors() method to get (count, color) tuples.
+    If the color is an integer (palette index), it converts it to an RGB tuple.
+    Sorts the colors by brightness (sum of RGB).
+    Pads with zeros if fewer than num_colors colors are present.
+    """
+    colors = quantized_image.getcolors(maxcolors=256)
+    if colors is None:
+        raise ValueError("Too many colors in quantized image")
+    
+    raw_palette = quantized_image.getpalette()
+    color_list = []
+    for count, col in colors:
+        # Convert palette index to RGB if needed.
+        if isinstance(col, int):
+            rgb = tuple(raw_palette[col*3: col*3+3])
+        else:
+            rgb = col
+        color_list.append(rgb)
+    
+    color_list.sort(key=lambda c: sum(c))
+    
+    if len(color_list) < num_colors:
+        pad_length = num_colors - len(color_list)
+        color_list.extend([(0, 0, 0)] * pad_length)
+    elif len(color_list) > num_colors:
+        color_list = color_list[:num_colors]
+    
+    feature = np.array(color_list).flatten()
+    return feature
+
+def process_textures(input_dir, output_dir, error_threshold, num_palette_groups=64, palette_colors=16):
+    accepted_features = []
+    accepted_files = []
+    non_paletteable_files = []
+
+    for filename in os.listdir(input_dir):
+        if filename.lower().endswith('.tga'):
+            filepath = os.path.join(input_dir, filename)
+            try:
+                original = Image.open(filepath).convert("RGB")
+                # Quantize image to a palette of palette_colors colors.
+                quantized = original.quantize(colors=palette_colors, method=Image.FASTOCTREE, dither=Image.NONE)
+                quantized_rgb = quantized.convert("RGB")
+                
+                original_arr = np.array(original, dtype=np.float32)
+                quantized_arr = np.array(quantized_rgb, dtype=np.float32)
+                
+                mse = compute_mse(original_arr, quantized_arr)
+                
+                if mse <= error_threshold:
+                    feature = extract_palette_feature(quantized, num_colors=palette_colors)
+                    accepted_features.append(feature)
+                    accepted_files.append(filepath)
+                    # print(f"File {filename} MSE={mse:.2f} -> paletteable")
+                else:
+                    non_paletteable_files.append(filepath)
+                    # print(f"File {filename} MSE={mse:.2f} -> non-paletteable")
+            except Exception as e:
+                print(f"Error processing {filename}: {e}")
+    
+    print(f"Total textures processed: {len(accepted_files) + len(non_paletteable_files)}")
+    print(f"Paletteable textures: {len(accepted_files)}")
+    print(f"Non-paletteable textures: {len(non_paletteable_files)}")
+    
+    if accepted_features:
+        features_array = np.array(accepted_features)
+        kmeans = KMeans(n_clusters=num_palette_groups, random_state=42)
+        cluster_labels = kmeans.fit_predict(features_array)
+        
+        for i in range(num_palette_groups):
+            cluster_dir = os.path.join(output_dir, f"cluster_{i}")
+            os.makedirs(cluster_dir, exist_ok=True)
+        
+        for filepath, label in zip(accepted_files, cluster_labels):
+            dest_dir = os.path.join(output_dir, f"cluster_{label}")
+            shutil.copy(filepath, dest_dir)
+    
+            cluster_filepath = filepath + ".cluster"
+            with open(cluster_filepath, "w") as cluster_file:
+                cluster_file.write(str(label))
+    
+    nonpal_dir = os.path.join(output_dir, "cluster_64")
+    os.makedirs(nonpal_dir, exist_ok=True)
+    for filepath in non_paletteable_files:
+        shutil.copy(filepath, nonpal_dir)
+
+        cluster_filepath = filepath + ".cluster"
+        with open(cluster_filepath, "w") as cluster_file:
+            cluster_file.write("64")
+    
+    print("Grouping complete.")
+
+def main():
+    parser = argparse.ArgumentParser(description="Group .tga textures into palette clusters and a non-paletteable group.")
+    parser.add_argument("input_dir", type=str, help="Path to folder containing .tga textures")
+    parser.add_argument("output_dir", type=str, help="Path to output folder for grouped textures")
+    parser.add_argument("--threshold", type=float, default=500.0,
+                        help="Error threshold for palette quality (default: 500.0)")
+    parser.add_argument("--palette_colors", type=int, default=16,
+                        help="Number of colors for palette quantization (default: 16)")
+    parser.add_argument("--num_palette_groups", type=int, default=64,
+                        help="Number of palette groups to form (default: 64)")
+    args = parser.parse_args()
+
+    os.makedirs(args.output_dir, exist_ok=True)
+    process_textures(args.input_dir, args.output_dir, args.threshold, args.num_palette_groups, args.palette_colors)
+
+if __name__ == "__main__":
+    main()

src/tools/texconv.cpp

@@ -53,6 +53,9 @@ uint32_t pvr_map32(uint32_t offset32) {return 0;}
 void Hackpresent() { }
 void re3RemoveLeastUsedModel() { assert(false); }
 void re3EmergencyRemoveModel() { assert(false); }
+void* re3StreamingAlloc(size_t sz) {
+	return RwMalloc(sz);
+}
 void RwTexDictionaryGtaStreamRead1(rw::Stream*){ assert(false); }
 void RwTexDictionaryGtaStreamRead2(rw::Stream*, rw::TexDictionary*) { assert(false);  }
 void pvr_ta_data(void* data, int size) {
@@ -64,6 +67,7 @@ namespace rw::dc {
 	extern int32 maxRasterHeight;
 	extern int32 downsampleMode;
 	extern int32 pvrEncoder;
+	extern const char* dstFile;
 }
 
 RwTexDictionary *LoadTxd(RwStream *stream);
@@ -304,6 +308,8 @@ int main(int argc, const char** argv) {
                         rw::dc::pvrEncoder = PVRTOOL;
                     } else if (strcmp(param, "pvrtex") == 0 || strcmp(param, "PVRTEX") == 0) {
                         rw::dc::pvrEncoder = PVRTEX;
+                    } else if (strcmp(param, "extract") == 0 || strcmp(param, "EXTRACT") == 0) {
+                        rw::dc::pvrEncoder = EXTRACT;
                     }
                 }
             }
@@ -341,6 +347,8 @@ int main(int argc, const char** argv) {
 	assert(RwEngineStart());
 	currentFile = argv[1];
 
+	rw::dc::dstFile = argv[2];
+
 	if (strstr(argv[1], ".txd") || strstr(argv[1], ".TXD")) {
 		auto stream = RwStreamOpen(rwSTREAMFILENAME, rwSTREAMREAD, argv[1]);
 		assert(stream && "failed to open input");
@@ -375,12 +383,14 @@ int main(int argc, const char** argv) {
 			}
 		}
 
-		auto streamOut = RwStreamOpen(rwSTREAMFILENAME, rwSTREAMWRITE, argv[2]);
-		assert(streamOut && "failed to open output");
-
-		StoreTxd(texDict, streamOut);
-
-		RwStreamClose(streamOut, nil);
+		if (rw::dc::pvrEncoder != EXTRACT) {
+			auto streamOut = RwStreamOpen(rwSTREAMFILENAME, rwSTREAMWRITE, argv[2]);
+			assert(streamOut && "failed to open output");
+	
+			StoreTxd(texDict, streamOut);
+	
+			RwStreamClose(streamOut, nil);
+		}
 	} else if (strstr(argv[1], ".dff") || strstr(argv[1], ".DFF")) {
 		rw::Texture::setLoadTextures(false);
 		rw::Texture::setCreateDummies(true);

vendor/emu/emu/window.cpp

@@ -125,6 +125,8 @@ void x11_window_create()
     x11_win = (void*)x11Window;
     x11_vis = (void*)x11Visual->visual;
 
+    delete x11Visual;
+
     x11_window_set_text("GTA3dc");
 }
 

vendor/librw/src/anim.cpp

@@ -221,6 +221,7 @@ AnimInterpolator::setCurrentAnim(Animation *anim)
 {
 	int32 i;
 	AnimInterpolatorInfo *interpInfo = anim->interpInfo;
+	assert(this->currentAnim == nil || this->currentAnim  == anim);
 	this->currentAnim = anim;
 	this->currentTime = 0.0f;
 	int32 maxkf = this->maxInterpKeyFrameSize;

vendor/librw/src/dc/tex-util.h

@@ -21,7 +21,8 @@ enum downsampleModes {
 
 enum pvrEncoders {
     PVRTOOL,
-    PVRTEX
+    PVRTEX,
+    EXTRACT
 };
 
 // TGA Header Struct
@@ -141,6 +142,10 @@ void loadPVR(char *fname, rw::Raster* raster, auto* natras, auto flags) {
             natras->pvr_flags |= PVR_TXRFMT_YUV422;
             rasterFmt = rw::Raster::C565;  // this is a bit of a hack
             break;
+        case 0x5: // PAL4
+            natras->pvr_flags |= PVR_TXRFMT_PAL4BPP;
+            rasterFmt = raster->format & 0x0F00; // perserve for blending reasons
+            break;
         default:
             assert(false && "Invalid texture format");
             break;
@@ -153,11 +158,11 @@ void loadPVR(char *fname, rw::Raster* raster, auto* natras, auto flags) {
     natras->texaddr = alloc_malloc(natras, natras->texsize);
     fread(natras->texaddr, 1, natras->texsize, tex); /* Read in the PVR texture data */
     texconvf("PVR TEXTURE READ: %ix%i, %i, %i\n", HDR.nWidth, HDR.nHeight, natras->texoffs, natras->texsize);
-    if (natras->texsize >= 256) {
-        assert((natras->texsize & 255) == 0);
-    } else {
-        assert((natras->texsize & 31) == 0);
-    }
+    // if (natras->texsize >= 256) {
+    //     assert((natras->texsize & 31) == 0);
+    // } else {
+    //     assert((natras->texsize & 31) == 0);
+    // }
 
     fclose(tex);
 }

vendor/pvrtex/file_pvr.c

@@ -53,7 +53,8 @@ void fPvrWrite(const PvrTexEncoder *pte, const char *outfname) {
 		}
 		
 		if (pteIsPalettized(pte))
-			ErrorExit(".PVR format does not support compressed palettized textures\n");
+			idxcnt/= 4;
+			// ErrorExit(".PVR format does not support compressed palettized textures\n");
 		// JP - Rectangle VQ certainly does work on real hardware
 		//if (pte->w != pte->h)
 		//	ErrorExit(".PVR format does not support non-square compressed textures\n");

vendor/pvrtex/main.c

@@ -12,7 +12,7 @@
 #include "file_pvr.h"
 #include "file_tex.h"
 
-int log_level = LOG_PROGRESS;
+int log_level = LOG_DEBUG;
 void pteLogLocV(unsigned level, const char *file, unsigned line, const char *fmt, va_list args) {
 	static const char * logtypes[] = {
 		[LOG_ALL] = "ALL",
@@ -163,10 +163,11 @@ int main(int argc, char **argv) {
 		{"mip-resize", 'R', OPTPARSE_OPTIONAL},
 		{"stride", 's', OPTPARSE_NONE},
 		{"edge", 'e', OPTPARSE_REQUIRED},
+		{"pallette", 'q', OPTPARSE_REQUIRED},
 		{0}
 	};
 	
-	#define MAX_FNAMES	11
+	#define MAX_FNAMES	1024
 	const char *fnames[MAX_FNAMES];
 	unsigned fname_cnt = 0;
 	const char *outname = "";
@@ -298,6 +299,23 @@ int main(int argc, char **argv) {
 					ErrorExit("invalid max palette size parameter (should be [1, 16] for 4bpp, or [1, 256] for 8bpp)\n");
 				}
 			break;
+		case 'q': {
+				pxlARGB8888 pal[16];
+				FILE* f = fopen(options.optarg, "rb");
+				assert(f);
+				assert(fseek(f, 8, SEEK_SET) == 0);
+				assert(fread(pal, 1, 4 * 16, f) == 4 * 16);
+				fclose(f);
+
+				pte.palette = malloc(16 * 4);
+				pte.palette_size = 16;
+				assert(pte.palette);
+
+				for (int i = 0; i < 16; i++) {
+					pte.palette[i] = pxlConvertARGB8888toABGR8888(pal[i]);
+				}
+			}
+			break;
 		default:
 			ErrorExit("%s\n", options.errmsg);
 		}
@@ -390,6 +408,34 @@ int main(int argc, char **argv) {
 			pte.edge_method = STBIR_EDGE_CLAMP;
 	}
 	
+	if (strcasecmp(extension, ".pal") == 0) {
+		//Generate palette
+		if (pte.pixel_format == PTE_PALETTE_4B || pte.pixel_format == PTE_PALETTE_8B) {
+			if (pte.pixel_format == PTE_PALETTE_8B) {
+				if (pte.palette_size == 0) {
+					pte.palette_size = 256;
+				} else if (pte.palette_size > 256) {
+					ErrorExit("palette size must be 256 or less for 8bpp textures\n");
+				}
+			} else if (pte.pixel_format == PTE_PALETTE_4B) {
+				if (pte.palette_size == 0) {
+					pte.palette_size = 16;
+				} else if (pte.palette_size > 16) {
+					ErrorExit("palette size must be 16 or less for 4bpp textures\n");
+				}
+			}
+			pteLog(LOG_PROGRESS, "Generating palette...\n");
+		} else {
+			assert(0 && ".pal file but no pal format");
+		}
+
+		pteGenerateGlobalPalette(&pte);
+		
+		fTexWritePalette(&pte, outname);
+		// pteFree(&pte); // its okay, this will leak a bit
+		return 0;
+	}
+
 	pteEncodeTexture(&pte);
 	
 	//Make preview

vendor/pvrtex/pixel.h

@@ -446,6 +446,15 @@ static inline pxlRGBA32 pxlConvertABGR8888toRGBA32(pxlABGR8888 color) {
 	ret.a = color.a;
 	return ret;
 }
+
+static inline pxlABGR8888 pxlConvertARGB8888toABGR8888(pxlARGB8888 color) {
+	pxlABGR8888 ret;
+	ret.r = color.r;
+	ret.g = color.g;
+	ret.b = color.b;
+	ret.a = color.a;
+	return ret;
+}
 static inline pxlARGB8888 pxlConvertABGR8888toARGB8888(pxlABGR8888 color) {
 	pxlARGB8888 ret;
 	ret.r = color.r;

vendor/pvrtex/pvr_texture.h

@@ -11,7 +11,7 @@ extern "C" {
 
 #define PVR_MAX_TEXTURE_WIDTH	1024
 #define PVR_MAX_TEXTURE_HEIGHT	1024
-#define PVR_MAX_MIPMAPS	(11)
+#define PVR_MAX_MIPMAPS	(1024)
 
 #define CHANNEL_CNT_ARGB	4
 #define VECTOR_W	2

vendor/pvrtex/pvr_texture_encoder.c

@@ -85,10 +85,10 @@ void pteLoadFromFiles(PvrTexEncoder *pte, const char **fnames, unsigned filecnt)
 				"When using custom mipmaps, the size of all levels must be a power of two"
 				" (resize is not supported). %s has a size of %ux%u\n", fnames[i], 
 				img->w, img->h);
-			ErrorExitOn(img->w != img->h, 
-				"When using custom mipmaps, all levels must be square"
-				" (resize is not supported). %s has a size of %ux%u\n", fnames[i], 
-				img->w, img->h);
+			// ErrorExitOn(img->w != img->h, 
+			// 	"When using custom mipmaps, all levels must be square"
+			// 	" (resize is not supported). %s has a size of %ux%u\n", fnames[i], 
+			// 	img->w, img->h);
 		}
 		
 		maxw = MAX(maxw, img->w);
@@ -288,7 +288,38 @@ void pteDitherRaws(PvrTexEncoder *pte, float dither_amt) {
 }
 
 
+void pteGenerateGlobalPalette(PvrTexEncoder *pte) {
+	assert(pte);
+	assert(pte->palette == NULL);
+	assert(pte->palette_size > 0);
+	assert(pte->palette_size <= 256);
+	
+	VQCompressor vqc;
+	vqcInit(&vqc, VQC_UINT8, 4, 1, pte->palette_size, pte->auto_small_vq);
+	vqcSetRGBAGamma(&vqc, pte->rgb_gamma, pte->alpha_gamma);
+	
+	//Add mipmaps to compressor input
+	for (int i = 0; i < pte->src_img_cnt; i++) {
+		uint32_t pixelcnt = pte->src_imgs[i].w * pte->src_imgs[i].h;
+		vqcAddPoints(&vqc, pte->src_imgs[i].pixels, pixelcnt);
+	}
+	
+	//Do compression and save resulting palette
+	vqcResults result = vqcCompress(&vqc, 8);
+	assert(result.codebook);
+	pte->palette = result.codebook;
+	free(result.indices);
+	
+	//~ for(unsigned i = 0; i < 256; i++) {
+		//~ printf("(%i, %i, %i) ", pte->palette[i].r, pte->palette[i].g, pte->palette[i].b, pte->palette[i].a);
+	//~ }
+}
+
 void pteGeneratePalette(PvrTexEncoder *pte) {
+	if (pte->palette) {
+		// already has palette preloaded
+		return;
+	}
 	assert(pte);
 	assert(pte->mip_cnt > 0);
 	assert(pte->mip_cnt <= PVR_MAX_MIPMAPS);

vendor/pvrtex/pvr_texture_encoder.h

@@ -202,6 +202,7 @@ int pteSetSize(PvrTexEncoder *pte);
 void pteSetCompressed(PvrTexEncoder *pte, int codebook_size);
 void pteGeneratePreviews(PvrTexEncoder *pte);
 void pteAutoSelectPixelFormat(PvrTexEncoder *pte);
+void pteGenerateGlobalPalette(PvrTexEncoder *pte);
 
 ///////////
 void ErrorExitOn(int cond, const char *fmt, ...);