Compare commits

..

3 Commits

Author SHA1 Message Date
204518302a update readme 2025-05-12 13:04:39 +07:00
f328c214b7 initial v1.3 2025-05-12 13:04:29 +07:00
021f5aa0cd adding submodule 2025-05-12 12:18:38 +07:00
8 changed files with 255 additions and 399 deletions

6
.gitmodules vendored Normal file
View File

@ -0,0 +1,6 @@
[submodule "external/FastLZ"]
path = external/FastLZ
url = https://github.com/ariya/FastLZ.git
[submodule "external/murmurhash"]
path = external/murmurhash
url = https://github.com/jwerle/murmurhash.c.git

View File

@ -12,24 +12,34 @@ project(${PROJECT_NAME} VERSION ${PROJECT_VERSION})
list(APPEND CMAKE_MODULE_PATH "/opt/cmake")
find_package(argh)
find_package(LIQ)
find_package(STB)
include(FetchContent)
FetchContent_Declare(
argparse
GIT_REPOSITORY https://github.com/p-ranav/argparse.git
)
FetchContent_MakeAvailable(argparse)
##############################################################
# yes... im using glob... dont judge me....
file(GLOB_RECURSE PROJECT_SOURCES CONFIGURE_DEPENDS "src/*.cpp")
file(GLOB VENDOR_SOURCES CONFIGURE_DEPENDS "")
file(GLOB EXTERNAL_SOURCES CONFIGURE_DEPENDS
"external/murmurhash/murmurhash.c"
"external/FastLZ/fastlz.c")
set(PROJECT_INCLUDE
"src"
${argh_INCLUDE_DIR}
${STB_INCLUDE_DIRS}
${LIQ_INCLUDE_DIRS})
${LIQ_INCLUDE_DIRS}
"external/ChernoTimer"
"external/murmurhash"
"external/FastLZ")
set(PROJECT_LIBRARY
argh
argparse
${LIQ_LIBRARIES})
set(PROJECT_DEFINITION
@ -37,7 +47,7 @@ set(PROJECT_DEFINITION
##############################################################
add_executable(${PROJECT_NAME} ${PROJECT_SOURCES} ${VENDOR_SOURCES})
add_executable(${PROJECT_NAME} ${PROJECT_SOURCES} ${EXTERNAL_SOURCES})
target_include_directories(${PROJECT_NAME} PUBLIC ${PROJECT_INCLUDE})
target_link_libraries(${PROJECT_NAME} PRIVATE ${PROJECT_LIBRARY})
target_compile_definitions(${PROJECT_NAME} PUBLIC ${PROJECT_DEFINITION})

1
external/FastLZ vendored Submodule

@ -0,0 +1 @@
Subproject commit b1342dabcf5257ab303743c9332fe75e9147a011

1
external/murmurhash vendored Submodule

@ -0,0 +1 @@
Subproject commit 10ba9c25abbcf8952b5f4abecf9bf4fc148e8e65

View File

@ -9,32 +9,23 @@ i tried to create header with the metadata information of the image
the idea is to simply load image data without worrying about image metadata.
the metadata contain basic stuff like width, height, format, etc...
there is also paletteId that tell which palette the image used.
usage: sillyimage <input image> [options]
color palette are embedded in the file.
a palette hash is provided. this can be used to comparing
whether another image uses the same palette
usage: sillyimage -i <input image> -o <output dir> [options]
options:
-o, --out
output directory
-f, --format
specify format {rgb256, rgb16, indexed4, indexed16, indexed256, indexed32a3, indexed8a5}
-c, --compress
specify compression {none, lzsswram, lzssvram, gzip}
-pid, --palette-id
specify the palette id
-n, --num
limit num color regardless of the format
-v, --verbose
enable verbose printing
-i, --input input files [required] [may be repeated]
-o, --out output directory [required]
-h, --help shows help message and exits
-v, --version prints version information and exits
-f, --format texture format { rgb256, rgb16, indexed4, indexed16, indexed256, indexed32a3, indexed8a5 } [default: "rgb16"]
binary format:
[string] sillyimg (0x676D69796C6C6973 in hex or 7452728928599042419 in decimal (uint64))
[uint8] version (current version is 0)
[uint8] version (current version is 13)
[uint8] format
0 - RGB256
1 - RGB16
@ -44,16 +35,14 @@ binary format:
5 - INDEXED32A3
6 - INDEXED8A5
7 - PALETTE16
[uint8] paletteId
[uint16] width
[uint16] height
[uint8] compression
0 - none
1 - lzss wram
2 - lzss vram
3 - gzip
[uint32] buffer length
[buffer]
[uint16] palette count
[uint32] palette hash
[uint32] compress size
[uint32] original size
[palette buffer]
[image buffer]
TODO:
[x] bitpacking for indexed format
@ -61,13 +50,14 @@ TODO:
[ ] convert image using predefined palette
[ ] output preview image
[ ] improve error handling
[ ] compression
[x] compression
dependecies:
argh (https://github.com/adishavit/argh)
stb_image, stb_image_writer (https://github.com/nothings/stb)
dependencies:
argparse (https://github.com/p-ranav/argparse)
stb_image (https://github.com/nothings/stb)
libimagequant (https://github.com/ImageOptim/libimagequant)
TImer.h from cherno (https://gist.github.com/TheCherno/b2c71c9291a4a1a29c889e76173c8d14)
murmurhash.c (https://github.com/jwerle/murmurhash.c)
Timer.h from cherno (https://gist.github.com/TheCherno/b2c71c9291a4a1a29c889e76173c8d14)
license:
GPL v3

View File

@ -16,18 +16,23 @@
#include <string>
#include <vector>
#pragma pack(push, 1) // Set the alignment to 1 byte (no padding)
// wporkaround surpress clangd pragma pack warning
// https://github.com/clangd/clangd/issues/1167
static_assert(true);
#pragma pack(push, 1)
struct Metadata {
uint64_t header = 0x676D69796C6C6973;
uint8_t version = 0;
uint8_t version = 13;
uint8_t format;
uint8_t paletteId;
uint16_t width;
uint16_t height;
uint8_t compression;
uint32_t length;
uint16_t palette_count;
uint32_t palette_hash;
uint32_t original_size;
uint32_t compress_size;
};
#pragma pack(pop) // Restore the previous alignment
#pragma pack(pop)
enum Format
{
@ -41,14 +46,6 @@ enum Format
Format_PALETTE_16,
};
enum Compress
{
Compress_NONE = 0,
Compress_LZSS_WRAM,
Compress_LZSS_VRAM,
Compress_GZIP,
};
struct Image {
int width, height, comp;
stbi_uc *data;
@ -79,27 +76,15 @@ struct Color {
uint8_t b;
};
struct Params {
std::vector<std::filesystem::path> input_list;
std::filesystem::path output_dir;
Format format;
Compress compress;
int paletteId;
int limitColor = 0;
bool verbose = false;
bool paletteOnly = false;
void Parse(int argc, char **argv);
void Convert();
void PrintHelp();
void Print(const std::string &msg);
struct Options {
std::vector<std::string> path_input;
std::string path_output;
std::string format;
};
void Verify();
void Convert();
uint16_t RGB16(uint8_t a, uint8_t r, uint8_t g, uint8_t b);
void GetIndexed(const int numColor, std::vector<Image> &images, std::vector<Color> &palette, std::vector<IndexedImage> &indexes);
void BitPacking(const uint8_t bpp, IndexedImage &img, uint32_t &length);
void WritePalette(const std::string &path, const uint8_t paletteId, std::vector<Color> &palette);
void WriteImage(const std::string &path, const Format &format, const uint16_t width, const uint16_t height, const Compress &compress, const uint8_t paletteId, void *buffer, const uint32_t length);
void QuantizeImage(const int numColor, std::vector<Image> &images, std::vector<Color> &palette, std::vector<IndexedImage> &indexes);
void BitPacking(const uint8_t bpp, IndexedImage &img, uint32_t &length);

View File

@ -8,7 +8,6 @@
* You should have received a copy of the GNU General Public License along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <cstdlib>
@ -22,33 +21,194 @@
#define STB_IMAGE_IMPLEMENTATION
#define STB_IMAGE_WRITE_IMPLEMENTATION
#include "argh.h"
#include <argparse/argparse.hpp>
#include "libimagequant.h"
#include "header.hpp"
#include "external/Timer.h"
#include "Timer.h"
#include "fastlz.h"
#include "murmurhash.h"
namespace fs = std::filesystem;
static void imagegen_job(uint8_t *image_output, uint8_t *image_indexed, const liq_palette *pal, int start, int end)
{
for (int i=start; i<end; i++)
{
auto &col = pal->entries[image_indexed[i]];
image_output[i*4+0] = col.r;
image_output[i*4+1] = col.g;
image_output[i*4+2] = col.b;
image_output[i*4+3] = col.a;
}
}
int main(int argc, char **argv)
{
try {
srand(time(0));
srand(time(0));
argparse::ArgumentParser program("sillyimage", "1.3");
Options opt;
// Register argument
{
program.add_argument("-i", "--input")
.help("input files")
.append()
.required()
.store_into(opt.path_input);
program.add_argument("-o", "--out")
.help("output directory")
.required()
.store_into(opt.path_output);
program.add_argument("-f", "--format")
.help("texture format { rgb256, rgb16, indexed4, indexed16, indexed256, indexed32a3, indexed8a5 }")
.default_value<std::string>("rgb16")
.choices("rgb256", "rgb16", "indexed4", "indexed16", "indexed256", "indexed32a3", "indexed8a5")
.nargs(1)
.store_into(opt.format);
}
// Parse argument
{
try
{
program.parse_args(argc, argv);
// input list
for (auto &i : opt.path_input)
if (i.empty() || !fs::exists(i))
throw std::runtime_error(std::format("file doesnt exsist: \"{}\"", i));
// output directory
if (opt.path_output.empty() || !fs::is_directory(opt.path_output))
throw std::runtime_error(std::format("invalid output directory: \"{}\"", opt.path_output));
}
catch(const std::exception &e)
{
std::cerr << std::format("Error: {}\n\n", e.what()) << program << "\n";
return 1;
}
}
// Convert
try {
if (opt.format == "rgb256")
{
for(auto &input : opt.path_input)
{
Image img;
img.Load(input);
Metadata meta;
meta.format = Format::Format_RGB_256;
meta.width = img.width;
meta.height = img.height;
meta.palette_count = 0;
meta.palette_hash = 0;
meta.original_size = img.width*img.height*img.comp;
uint8_t compress[uint64_t(meta.original_size*1.5)];
meta.compress_size = fastlz_compress_level(1, img.data, meta.original_size, compress);
auto output = fs::path(opt.path_output) / std::format("{}.sillyimg", fs::path(input).stem().string());
std::ofstream out(output, std::ios::binary);
out.write(reinterpret_cast<const char*>(&meta), sizeof(meta));
out.write(reinterpret_cast<const char*>(compress), meta.compress_size);
out.close();
}
}
if (opt.format == "rgb16")
{
for(auto &input : opt.path_input)
{
Image img;
img.Load(input);
uint16_t img16[img.width*img.height];
// convert into RGB16 color
for (int i=0; i<img.width*img.height; i++)
{
uint8_t a = img.data[i*4+3];
uint8_t r = img.data[i*4+0];
uint8_t g = img.data[i*4+1];
uint8_t b = img.data[i*4+2];
img16[i] = RGB16(a, r, g, b);
}
Metadata meta;
meta.format = Format::Format_RGB_16;
meta.width = img.width;
meta.height = img.height;
meta.palette_count = 0;
meta.palette_hash = 0;
meta.original_size = img.width*img.height*2;
uint8_t compress[uint64_t(meta.original_size*1.5)];
meta.compress_size = fastlz_compress_level(1, img16, meta.original_size, compress);
auto output = fs::path(opt.path_output) / std::format("{}.sillyimg", fs::path(input).stem().string());
std::ofstream out(output, std::ios::binary);
out.write(reinterpret_cast<const char*>(&meta), sizeof(meta));
out.write(reinterpret_cast<const char*>(compress), meta.compress_size);
out.close();
}
}
if (opt.format == "indexed4" || opt.format == "indexed16" || opt.format == "indexed256")
{
uint16_t numcol;
uint16_t bpp;
if (opt.format == "indexed4")
{
numcol = 4;
bpp = 2;
}
if (opt.format == "indexed16")
{
numcol = 16;
bpp = 4;
}
if (opt.format == "indexed256")
{
numcol = 256;
bpp = 8;
}
std::vector<Color> palette;
std::vector<Image> images(opt.path_input.size());
std::vector<IndexedImage> indexes(opt.path_input.size());
for (int i=0; i<images.size(); i++)
images[i].Load(opt.path_input[i]);
QuantizeImage(numcol, images, palette, indexes);
uint16_t palette16[palette.size()];
for (const auto &pal : palette)
uint16_t rgb16 = RGB16(pal.a, pal.r, pal.g, pal.b);
for (int i=0; i<indexes.size(); i++)
{
uint32_t length = indexes[i].width*indexes[i].height;
if (bpp < 8) BitPacking(bpp, indexes[i], length);
Metadata meta;
meta.format = Format::Format_RGB_16;
meta.width = indexes[i].width;
meta.height = indexes[i].height;
meta.palette_count = numcol;
meta.palette_hash = murmurhash(reinterpret_cast<const char*>(palette16), sizeof(uint16_t) * palette.size(), 1);
meta.original_size = length;
uint8_t compress[uint64_t(meta.original_size*1.5)];
meta.compress_size = fastlz_compress_level(1, indexes[i].data, meta.original_size, compress);
auto output = fs::path(opt.path_output) / std::format("{}.sillyimg", fs::path(opt.path_input[i]).stem().string());
std::ofstream out(output, std::ios::binary);
out.write(reinterpret_cast<const char*>(&meta), sizeof(meta));
out.write(reinterpret_cast<const char*>(palette16), sizeof(uint16_t) * palette.size());
out.write(reinterpret_cast<const char*>(compress), meta.compress_size);
out.close();
}
}
if (opt.format == "indexed32a3" || opt.format == "indexed8a5")
throw std::runtime_error("format not supported yet!");
Params params;
params.Parse(argc, argv);
params.Convert();
}
catch(const std::exception &e)
{
@ -61,314 +221,16 @@ int main(int argc, char **argv)
return 1;
}
std::cout << "done\n";
return 0;
}
void Params::Parse(int argc, char **argv)
{
argh::parser cmdl({
"-f", "--format",
"-o", "--out",
"-c", "--compres",
"-p", "--palette",
"-pid", "--palette-id",
});
cmdl.parse(argc, argv);
if (cmdl.size() == 1) PrintHelp();
verbose = cmdl[{ "-v", "--verbose" }];
paletteOnly = cmdl[{ "-po", "--palette-only" }];
cmdl({"-pid", "--palette-id"}, rand() % 256) >> paletteId;
// file input checking
{
std::stringstream ss(cmdl(1, "").str());
std::string token;
while (getline(ss, token, ','))
input_list.push_back(token);
for (auto &i : input_list)
if (i.empty() || !fs::exists(i))
throw std::runtime_error(std::format("file doesnt exsist: {}", i.string()));
}
// output directory checking
{
output_dir = cmdl({"-o", "--out"}, "").str();
if (output_dir.empty() || !fs::is_directory(output_dir))
throw std::runtime_error(std::format("invalid output directory: {}", output_dir.string()));
}
// format checking
{
std::string temp = cmdl({"-f", "--format"}, "rgb16").str();
if (temp == "rgb256" || temp == "rgb255" || temp == "rgb")
format = Format_RGB_256;
else if (temp == "rgb16" || temp == "rgb15")
format = Format_RGB_16;
else if (temp == "indexed4" || temp == "4")
format = Format_INDEXED_4;
else if (temp == "indexed16" || temp == "16")
format = Format_INDEXED_16;
else if (temp == "indexed256" || temp == "256")
format = Format_INDEXED_256;
else if (temp == "indexed32a3" || temp == "i5a3")
format = Format_INDEXED_32A3;
else if (temp == "indexed8a5" || temp == "i3a5")
format = Format_INDEXED_8A5;
else throw std::runtime_error(std::format("Invalid format: {}", temp));
}
// compress checking
{
std::string temp = cmdl({"-c", "--compress"}, "none").str();
if (temp == "none")
compress = Compress_NONE;
else if (temp == "lzsswram" || temp == "lzss")
compress = Compress_LZSS_WRAM;
else if (temp == "lzssvram")
compress = Compress_LZSS_VRAM;
else if (temp == "gzip")
compress = Compress_GZIP;
else throw std::runtime_error(std::format("Invalid compress: {}", temp));
}
// num check
{
cmdl({"-n", "--num"}, 0) >> limitColor;
if (limitColor > 0)
{
switch (format) {
case Format_RGB_256: limitColor = std::clamp(limitColor, 0, 256); break;
case Format_RGB_16: limitColor = std::clamp(limitColor, 0, 16); break;
case Format_INDEXED_4: limitColor = std::clamp(limitColor, 0, 4); break;
case Format_INDEXED_16: limitColor = std::clamp(limitColor, 0, 16); break;
case Format_INDEXED_256: limitColor = std::clamp(limitColor, 0, 256); break;
case Format_INDEXED_32A3: limitColor = std::clamp(limitColor, 0, 32); break;
case Format_INDEXED_8A5: limitColor = std::clamp(limitColor, 0, 8); break;
case Format_PALETTE_16: break;
}
}
}
Print("\n==============\n");
Print("input:\n");
for (auto &i : input_list)
Print(std::format(" {}\n", i.string()));
Print(std::format("output dir: {}\n", output_dir.string()));
Print(std::format("format: {}\n", static_cast<int>(format)));
Print(std::format("compress: {}\n", static_cast<int>(compress)));
Print(std::format("paletteId: {}\n", static_cast<int>(paletteId)));
if (limitColor > 0)
Print(std::format("limitColor: {}\n", limitColor));
Print("==============\n\n");
}
void Params::Convert()
{
switch (format) {
case Format_RGB_256:
{
for (int i=0; i<input_list.size(); i++)
{
Image img;
img.Load(input_list[i]);
WriteImage(
(output_dir/input_list[i].stem()).string() + "_img.bin",
format, img.width, img.height, compress, 0,
img.data, img.width*img.height*4);
}
break;
}
case Format_RGB_16:
{
for (int i=0; i<input_list.size(); i++)
{
Image img;
img.Load(input_list[i]);
uint16_t *img16 = new uint16_t[img.width*img.height];
for (int i2=0; i2<img.width*img.height; i2++)
img16[i2*2] = RGB16(img.data[i2*4+3], img.data[i2*4+0], img.data[i2*4+1], img.data[i2*4+2]);
WriteImage(
output_dir / std::format("{}_img.bin", input_list[i].stem().string()),
format, img.width, img.height, compress, 0,
img16, img.width*img.height*2);
delete[] img16;
}
break;
}
// TODO: wrap indexed into function to reduce code repetition
case Format_INDEXED_4:
{
std::vector<Color> palette;
std::vector<Image> images(input_list.size());
std::vector<IndexedImage> indexes(input_list.size());
for (int i=0; i<images.size(); i++)
images[i].Load(input_list[i]);
GetIndexed(4, images, palette, indexes);
WritePalette(output_dir / std::format("{}_pal.bin", paletteId), paletteId, palette);
for (int i=0; i<indexes.size(); i++)
{
uint32_t length;
BitPacking(2, indexes[i], length);
WriteImage(
output_dir / std::format("{}_img.bin", input_list[i].stem().string()),
format, indexes[i].width, indexes[i].height, compress, paletteId,
indexes[i].data, length);
}
break;
}
case Format_INDEXED_16:
{
std::vector<Color> palette;
std::vector<Image> images(input_list.size());
std::vector<IndexedImage> indexes(input_list.size());
for (int i=0; i<images.size(); i++)
images[i].Load(input_list[i]);
GetIndexed(16, images, palette, indexes);
WritePalette(output_dir / std::format("{}_pal.bin", paletteId), paletteId, palette);
for (int i=0; i<indexes.size(); i++)
{
uint32_t length;
BitPacking(4, indexes[i], length);
WriteImage(
output_dir / std::format("{}_img.bin", input_list[i].stem().string()),
format, indexes[i].width, indexes[i].height, compress, paletteId,
indexes[i].data, length);
}
break;
}
case Format_INDEXED_256:
{
std::vector<Color> palette;
std::vector<Image> images(input_list.size());
std::vector<IndexedImage> indexes(input_list.size());
for (int i=0; i<images.size(); i++)
images[i].Load(input_list[i]);
GetIndexed(256, images, palette, indexes);
WritePalette(output_dir / std::format("{}_pal.bin", paletteId), paletteId, palette);
for (int i=0; i<indexes.size(); i++)
WriteImage(
output_dir / std::format("{}_img.bin", input_list[i].stem().string()),
format, indexes[i].width, indexes[i].height, compress, paletteId,
indexes[i].data, indexes[i].width*indexes[i].height);
break;
}
case Format_INDEXED_32A3:
case Format_INDEXED_8A5:
case Format_PALETTE_16:
throw std::runtime_error("currently not implemented yet!");
break;
}
}
void Params::PrintHelp()
{
// std::cout << "quantization <input path> [output path]" << "\n\n";
}
void Params::Print(const std::string &msg)
{
if (verbose) std::cout << msg;
}
// ----
void WritePalette(const std::string &path, const uint8_t paletteId, std::vector<Color> &palette)
{
Metadata meta;
meta.format = Format_PALETTE_16;
meta.paletteId = paletteId;
meta.width = palette.size();
meta.height = palette.size();
meta.compression = Compress_NONE;
meta.length = palette.size()*2;
std::ofstream out(path, std::ios::binary);
out.write(reinterpret_cast<const char*>(&meta), sizeof(meta));
for (const auto &i : palette)
{
uint16_t rgb16 = RGB16(i.a, i.r, i.g, i.b);
out.write(reinterpret_cast<const char*>(&rgb16), sizeof(rgb16));
}
out.close();
}
void WriteImage(const std::string &path, const Format &format, const uint16_t width, const uint16_t height, const Compress &compress, const uint8_t paletteId, void *buffer, const uint32_t length)
{
Metadata meta;
meta.format = format;
meta.paletteId = paletteId;
meta.width = width;
meta.height = height;
meta.compression = Compress_NONE;
meta.length = length;
std::ofstream out(path, std::ios::binary);
out.write(reinterpret_cast<const char*>(&meta), sizeof(meta));
out.write(reinterpret_cast<const char*>(buffer), length);
out.close();
}
// ----
// // ----
uint16_t RGB16(uint8_t a, uint8_t r, uint8_t g, uint8_t b)
{
// TODO: alpha threshold
uint16_t a1 = a & 0x01;
uint16_t r5 = (r >> 3) & 0x1F;
uint16_t g5 = (g >> 3) & 0x1F;
@ -377,7 +239,7 @@ uint16_t RGB16(uint8_t a, uint8_t r, uint8_t g, uint8_t b)
return (a1 << 15) | (r5) | (g5 << 5) | (b5 << 10);
}
void GetIndexed(const int numColor, std::vector<Image> &images, std::vector<Color> &palette, std::vector<IndexedImage> &indexes)
void QuantizeImage(const int numColor, std::vector<Image> &images, std::vector<Color> &palette, std::vector<IndexedImage> &indexes)
{
liq_attr *liq_attr;
liq_result *liq_result;
@ -420,6 +282,7 @@ void GetIndexed(const int numColor, std::vector<Image> &images, std::vector<Colo
for (auto &i : liq_images)
liq_image_destroy(i);
liq_attr_destroy(liq_attr);
liq_histogram_destroy(liq_histogram);
liq_result_destroy(liq_result);
@ -448,7 +311,7 @@ void BitPacking(const uint8_t bpp, IndexedImage &img, uint32_t &length)
img.data = buffer;
}
// ----
// // ----
Image::Image(const std::filesystem::path &path)
{
@ -466,7 +329,7 @@ void Image::Load(const std::filesystem::path &path)
if (data == nullptr) throw std::runtime_error(stbi_failure_reason());
}
// ----
// // ----
IndexedImage::IndexedImage(int width, int height)
: width(width), height(height), data(nullptr)