#include "bitpacking.h"

#include <cstdint>
#include <type_traits>

namespace duckdb_fastpforlib {
namespace internal {

// Used for uint8_t, uint16_t and uint32_t
template <uint8_t DELTA, uint8_t SHR, class TYPE, uint8_t TYPE_SIZE = sizeof(TYPE) * 8>
typename std::enable_if<(DELTA + SHR) < TYPE_SIZE>::type unpack_single_out(const TYPE *__restrict in,
                                                                           TYPE *__restrict out) {
	*out = ((*in) >> SHR) % (1 << DELTA);
}

// Used for uint8_t, uint16_t and uint32_t
template <uint8_t DELTA, uint8_t SHR, class TYPE, uint8_t TYPE_SIZE = sizeof(TYPE) * 8>
typename std::enable_if<(DELTA + SHR) >= TYPE_SIZE>::type unpack_single_out(const TYPE *__restrict &in,
                                                                            TYPE *__restrict out) {
	*out = (*in) >> SHR;
	++in;

	static const TYPE NEXT_SHR = SHR + DELTA - TYPE_SIZE;
	*out |= ((*in) % (1U << NEXT_SHR)) << (TYPE_SIZE - SHR);
}

template <uint8_t DELTA, uint8_t SHR>
typename std::enable_if<(DELTA + SHR) < 32>::type unpack_single_out(const uint32_t *__restrict in,
                                                                    uint64_t *__restrict out) {
	*out = ((static_cast<uint64_t>(*in)) >> SHR) % (1ULL << DELTA);
}

template <uint8_t DELTA, uint8_t SHR>
typename std::enable_if<(DELTA + SHR) >= 32 && (DELTA + SHR) < 64>::type
unpack_single_out(const uint32_t *__restrict &in, uint64_t *__restrict out) {
	*out = static_cast<uint64_t>(*in) >> SHR;
	++in;
	if (DELTA + SHR > 32) {
		static const uint8_t NEXT_SHR = SHR + DELTA - 32;
		*out |= static_cast<uint64_t>((*in) % (1U << NEXT_SHR)) << (32 - SHR);
	}
}

template <uint8_t DELTA, uint8_t SHR>
typename std::enable_if<(DELTA + SHR) >= 64>::type unpack_single_out(const uint32_t *__restrict &in,
                                                                     uint64_t *__restrict out) {
	*out = static_cast<uint64_t>(*in) >> SHR;
	++in;

	*out |= static_cast<uint64_t>(*in) << (32 - SHR);
	++in;

	if (DELTA + SHR > 64) {
		static const uint8_t NEXT_SHR = DELTA + SHR - 64;
		*out |= static_cast<uint64_t>((*in) % (1U << NEXT_SHR)) << (64 - SHR);
	}
}

// Used for uint8_t, uint16_t and uint32_t
template <class TYPE, uint16_t DELTA, uint16_t SHL, TYPE MASK, uint8_t TYPE_SIZE = sizeof(TYPE) * 8>
    typename std::enable_if < DELTA + SHL<TYPE_SIZE>::type pack_single_in(const TYPE in, TYPE *__restrict out) {
	if (SHL == 0) {
		*out = in & MASK;
	} else {
		*out |= (in & MASK) << SHL;
	}
}

// Used for uint8_t, uint16_t and uint32_t
template <class TYPE, uint16_t DELTA, uint16_t SHL, TYPE MASK, uint8_t TYPE_SIZE = sizeof(TYPE) * 8>
typename std::enable_if<DELTA + SHL >= TYPE_SIZE>::type pack_single_in(const TYPE in, TYPE *__restrict &out) {
	*out |= in << SHL;
	++out;

	if (DELTA + SHL > TYPE_SIZE) {
		*out = (in & MASK) >> (TYPE_SIZE - SHL);
	}
}

template <uint16_t DELTA, uint16_t SHL, uint64_t MASK>
    typename std::enable_if < DELTA + SHL<32>::type pack_single_in64(const uint64_t in, uint32_t *__restrict out) {
	if (SHL == 0) {
		*out = static_cast<uint32_t>(in & MASK);
	} else {
		*out |= (in & MASK) << SHL;
	}
}
template <uint16_t DELTA, uint16_t SHL, uint64_t MASK>
        typename std::enable_if < DELTA + SHL >= 32 &&
    DELTA + SHL<64>::type pack_single_in64(const uint64_t in, uint32_t *__restrict &out) {
	if (SHL == 0) {
		*out = static_cast<uint32_t>(in & MASK);
	} else {
		*out |= (in & MASK) << SHL;
	}

	++out;

	if (DELTA + SHL > 32) {
		*out = static_cast<uint32_t>((in & MASK) >> (32 - SHL));
	}
}
template <uint16_t DELTA, uint16_t SHL, uint64_t MASK>
typename std::enable_if<DELTA + SHL >= 64>::type pack_single_in64(const uint64_t in, uint32_t *__restrict &out) {
	*out |= in << SHL;
	++out;

	*out = static_cast<uint32_t>((in & MASK) >> (32 - SHL));
	++out;

	if (DELTA + SHL > 64) {
		*out = (in & MASK) >> (64 - SHL);
	}
}
template <uint16_t DELTA, uint16_t OINDEX = 0>
struct Unroller8 {
	static void Unpack(const uint8_t *__restrict &in, uint8_t *__restrict out) {
		unpack_single_out<DELTA, (DELTA * OINDEX) % 8>(in, out + OINDEX);

		Unroller8<DELTA, OINDEX + 1>::Unpack(in, out);
	}

	static void Pack(const uint8_t *__restrict in, uint8_t *__restrict out) {
		pack_single_in<uint8_t, DELTA, (DELTA * OINDEX) % 8, (1U << DELTA) - 1>(in[OINDEX], out);

		Unroller8<DELTA, OINDEX + 1>::Pack(in, out);
	}

};\
template <uint16_t DELTA>
struct Unroller8<DELTA, 7> {
	enum { SHIFT = (DELTA * 7) % 8 };

	static void Unpack(const uint8_t *__restrict in, uint8_t *__restrict out) {
		out[7] = (*in) >> SHIFT;
	}

	static void Pack(const uint8_t *__restrict in, uint8_t *__restrict out) {
		*out |= (in[7] << SHIFT);
	}
};

template <uint16_t DELTA, uint16_t OINDEX = 0>
struct Unroller16 {
	static void Unpack(const uint16_t *__restrict &in, uint16_t *__restrict out) {
		unpack_single_out<DELTA, (DELTA * OINDEX) % 16>(in, out + OINDEX);

		Unroller16<DELTA, OINDEX + 1>::Unpack(in, out);
	}

	static void Pack(const uint16_t *__restrict in, uint16_t *__restrict out) {
		pack_single_in<uint16_t, DELTA, (DELTA * OINDEX) % 16, (1U << DELTA) - 1>(in[OINDEX], out);

		Unroller16<DELTA, OINDEX + 1>::Pack(in, out);
	}

};

template <uint16_t DELTA>
struct Unroller16<DELTA, 15> {
	enum { SHIFT = (DELTA * 15) % 16 };

	static void Unpack(const uint16_t *__restrict in, uint16_t *__restrict out) {
		out[15] = (*in) >> SHIFT;
	}

	static void Pack(const uint16_t *__restrict in, uint16_t *__restrict out) {
		*out |= (in[15] << SHIFT);
	}
};

template <uint16_t DELTA, uint16_t OINDEX = 0>
struct Unroller {
	static void Unpack(const uint32_t *__restrict &in, uint32_t *__restrict out) {
		unpack_single_out<DELTA, (DELTA * OINDEX) % 32>(in, out + OINDEX);

		Unroller<DELTA, OINDEX + 1>::Unpack(in, out);
	}

	static void Unpack(const uint32_t *__restrict &in, uint64_t *__restrict out) {
		unpack_single_out<DELTA, (DELTA * OINDEX) % 32>(in, out + OINDEX);

		Unroller<DELTA, OINDEX + 1>::Unpack(in, out);
	}

	static void Pack(const uint32_t *__restrict in, uint32_t *__restrict out) {
		pack_single_in<uint32_t, DELTA, (DELTA * OINDEX) % 32, (1U << DELTA) - 1>(in[OINDEX], out);

		Unroller<DELTA, OINDEX + 1>::Pack(in, out);
	}

	static void Pack(const uint64_t *__restrict in, uint32_t *__restrict out) {
		pack_single_in64<DELTA, (DELTA * OINDEX) % 32, (1ULL << DELTA) - 1>(in[OINDEX], out);

		Unroller<DELTA, OINDEX + 1>::Pack(in, out);
	}
};

template <uint16_t DELTA>
struct Unroller<DELTA, 31> {
	enum { SHIFT = (DELTA * 31) % 32 };

	static void Unpack(const uint32_t *__restrict in, uint32_t *__restrict out) {
		out[31] = (*in) >> SHIFT;
	}

	static void Unpack(const uint32_t *__restrict in, uint64_t *__restrict out) {
		out[31] = (*in) >> SHIFT;
		if (DELTA > 32) {
			++in;
			out[31] |= static_cast<uint64_t>(*in) << (32 - SHIFT);
		}
	}

	static void Pack(const uint32_t *__restrict in, uint32_t *__restrict out) {
		*out |= (in[31] << SHIFT);
	}

	static void Pack(const uint64_t *__restrict in, uint32_t *__restrict out) {
		*out |= (in[31] << SHIFT);
		if (DELTA > 32) {
			++out;
			*out = static_cast<uint32_t>(in[31] >> (32 - SHIFT));
		}
	}
};

// Special cases
void __fastunpack0(const uint8_t *__restrict, uint8_t *__restrict out) {
	for (uint8_t i = 0; i < 8; ++i)
		*(out++) = 0;
}

void __fastunpack0(const uint16_t *__restrict, uint16_t *__restrict out) {
	for (uint16_t i = 0; i < 16; ++i)
		*(out++) = 0;
}

void __fastunpack0(const uint32_t *__restrict, uint32_t *__restrict out) {
	for (uint32_t i = 0; i < 32; ++i)
		*(out++) = 0;
}

void __fastunpack0(const uint32_t *__restrict, uint64_t *__restrict out) {
	for (uint32_t i = 0; i < 32; ++i)
		*(out++) = 0;
}

void __fastpack0(const uint8_t *__restrict, uint8_t *__restrict) {
}
void __fastpack0(const uint16_t *__restrict, uint16_t *__restrict) {
}
void __fastpack0(const uint32_t *__restrict, uint32_t *__restrict) {
}
void __fastpack0(const uint64_t *__restrict, uint32_t *__restrict) {
}

// fastunpack for 8 bits
void __fastunpack1(const uint8_t *__restrict in, uint8_t *__restrict out) {
	Unroller8<1>::Unpack(in, out);
}

void __fastunpack2(const uint8_t *__restrict in, uint8_t *__restrict out) {
	Unroller8<2>::Unpack(in, out);
}

void __fastunpack3(const uint8_t *__restrict in, uint8_t *__restrict out) {
	Unroller8<3>::Unpack(in, out);
}

void __fastunpack4(const uint8_t *__restrict in, uint8_t *__restrict out) {
	for (uint8_t outer = 0; outer < 4; ++outer) {
		for (uint8_t inwordpointer = 0; inwordpointer < 8; inwordpointer += 4)
			*(out++) = ((*in) >> inwordpointer) % (1U << 4);
		++in;
	}
}

void __fastunpack5(const uint8_t *__restrict in, uint8_t *__restrict out) {
	Unroller8<5>::Unpack(in, out);
}

void __fastunpack6(const uint8_t *__restrict in, uint8_t *__restrict out) {
	Unroller8<6>::Unpack(in, out);
}

void __fastunpack7(const uint8_t *__restrict in, uint8_t *__restrict out) {
	Unroller8<7>::Unpack(in, out);
}

void __fastunpack8(const uint8_t *__restrict in, uint8_t *__restrict out) {
	for (int k = 0; k < 8; ++k)
		out[k] = in[k];
}


// fastunpack for 16 bits
void __fastunpack1(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<1>::Unpack(in, out);
}

void __fastunpack2(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<2>::Unpack(in, out);
}

void __fastunpack3(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<3>::Unpack(in, out);
}

void __fastunpack4(const uint16_t *__restrict in, uint16_t *__restrict out) {
	for (uint16_t outer = 0; outer < 4; ++outer) {
		for (uint16_t inwordpointer = 0; inwordpointer < 16; inwordpointer += 4)
			*(out++) = ((*in) >> inwordpointer) % (1U << 4);
		++in;
	}
}

void __fastunpack5(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<5>::Unpack(in, out);
}

void __fastunpack6(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<6>::Unpack(in, out);
}

void __fastunpack7(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<7>::Unpack(in, out);
}

void __fastunpack8(const uint16_t *__restrict in, uint16_t *__restrict out) {
	for (uint16_t outer = 0; outer < 8; ++outer) {
		for (uint16_t inwordpointer = 0; inwordpointer < 16; inwordpointer += 8)
			*(out++) = ((*in) >> inwordpointer) % (1U << 8);
		++in;
	}
}

void __fastunpack9(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<9>::Unpack(in, out);
}

void __fastunpack10(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<10>::Unpack(in, out);
}

void __fastunpack11(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<11>::Unpack(in, out);
}

void __fastunpack12(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<12>::Unpack(in, out);
}

void __fastunpack13(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<13>::Unpack(in, out);
}

void __fastunpack14(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<14>::Unpack(in, out);
}

void __fastunpack15(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<15>::Unpack(in, out);
}

void __fastunpack16(const uint16_t *__restrict in, uint16_t *__restrict out) {
	for (int k = 0; k < 16; ++k)
		out[k] = in[k];
}

// fastunpack for 32 bits
void __fastunpack1(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<1>::Unpack(in, out);
}

void __fastunpack2(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<2>::Unpack(in, out);
}

void __fastunpack3(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<3>::Unpack(in, out);
}

void __fastunpack4(const uint32_t *__restrict in, uint32_t *__restrict out) {
	for (uint32_t outer = 0; outer < 4; ++outer) {
		for (uint32_t inwordpointer = 0; inwordpointer < 32; inwordpointer += 4)
			*(out++) = ((*in) >> inwordpointer) % (1U << 4);
		++in;
	}
}

void __fastunpack5(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<5>::Unpack(in, out);
}

void __fastunpack6(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<6>::Unpack(in, out);
}

void __fastunpack7(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<7>::Unpack(in, out);
}

void __fastunpack8(const uint32_t *__restrict in, uint32_t *__restrict out) {
	for (uint32_t outer = 0; outer < 8; ++outer) {
		for (uint32_t inwordpointer = 0; inwordpointer < 32; inwordpointer += 8)
			*(out++) = ((*in) >> inwordpointer) % (1U << 8);
		++in;
	}
}

void __fastunpack9(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<9>::Unpack(in, out);
}

void __fastunpack10(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<10>::Unpack(in, out);
}

void __fastunpack11(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<11>::Unpack(in, out);
}

void __fastunpack12(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<12>::Unpack(in, out);
}

void __fastunpack13(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<13>::Unpack(in, out);
}

void __fastunpack14(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<14>::Unpack(in, out);
}

void __fastunpack15(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<15>::Unpack(in, out);
}

void __fastunpack16(const uint32_t *__restrict in, uint32_t *__restrict out) {
	for (uint32_t outer = 0; outer < 16; ++outer) {
		for (uint32_t inwordpointer = 0; inwordpointer < 32; inwordpointer += 16)
			*(out++) = ((*in) >> inwordpointer) % (1U << 16);
		++in;
	}
}

void __fastunpack17(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<17>::Unpack(in, out);
}

void __fastunpack18(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<18>::Unpack(in, out);
}

void __fastunpack19(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<19>::Unpack(in, out);
}

void __fastunpack20(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<20>::Unpack(in, out);
}

void __fastunpack21(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<21>::Unpack(in, out);
}

void __fastunpack22(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<22>::Unpack(in, out);
}

void __fastunpack23(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<23>::Unpack(in, out);
}

void __fastunpack24(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<24>::Unpack(in, out);
}

void __fastunpack25(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<25>::Unpack(in, out);
}

void __fastunpack26(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<26>::Unpack(in, out);
}

void __fastunpack27(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<27>::Unpack(in, out);
}

void __fastunpack28(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<28>::Unpack(in, out);
}

void __fastunpack29(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<29>::Unpack(in, out);
}

void __fastunpack30(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<30>::Unpack(in, out);
}

void __fastunpack31(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<31>::Unpack(in, out);
}

void __fastunpack32(const uint32_t *__restrict in, uint32_t *__restrict out) {
	for (int k = 0; k < 32; ++k)
		out[k] = in[k];
}

// fastupack for 64 bits
void __fastunpack1(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<1>::Unpack(in, out);
}

void __fastunpack2(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<2>::Unpack(in, out);
}

void __fastunpack3(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<3>::Unpack(in, out);
}

void __fastunpack4(const uint32_t *__restrict in, uint64_t *__restrict out) {
	for (uint32_t outer = 0; outer < 4; ++outer) {
		for (uint32_t inwordpointer = 0; inwordpointer < 32; inwordpointer += 4)
			*(out++) = ((*in) >> inwordpointer) % (1U << 4);
		++in;
	}
}

void __fastunpack5(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<5>::Unpack(in, out);
}

void __fastunpack6(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<6>::Unpack(in, out);
}

void __fastunpack7(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<7>::Unpack(in, out);
}

void __fastunpack8(const uint32_t *__restrict in, uint64_t *__restrict out) {
	for (uint32_t outer = 0; outer < 8; ++outer) {
		for (uint32_t inwordpointer = 0; inwordpointer < 32; inwordpointer += 8) {
			*(out++) = ((*in) >> inwordpointer) % (1U << 8);
		}
		++in;
	}
}

void __fastunpack9(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<9>::Unpack(in, out);
}

void __fastunpack10(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<10>::Unpack(in, out);
}

void __fastunpack11(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<11>::Unpack(in, out);
}

void __fastunpack12(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<12>::Unpack(in, out);
}

void __fastunpack13(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<13>::Unpack(in, out);
}

void __fastunpack14(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<14>::Unpack(in, out);
}

void __fastunpack15(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<15>::Unpack(in, out);
}

void __fastunpack16(const uint32_t *__restrict in, uint64_t *__restrict out) {
	for (uint32_t outer = 0; outer < 16; ++outer) {
		for (uint32_t inwordpointer = 0; inwordpointer < 32; inwordpointer += 16)
			*(out++) = ((*in) >> inwordpointer) % (1U << 16);
		++in;
	}
}

void __fastunpack17(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<17>::Unpack(in, out);
}

void __fastunpack18(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<18>::Unpack(in, out);
}

void __fastunpack19(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<19>::Unpack(in, out);
}

void __fastunpack20(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<20>::Unpack(in, out);
}

void __fastunpack21(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<21>::Unpack(in, out);
}

void __fastunpack22(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<22>::Unpack(in, out);
}

void __fastunpack23(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<23>::Unpack(in, out);
}

void __fastunpack24(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<24>::Unpack(in, out);
}

void __fastunpack25(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<25>::Unpack(in, out);
}

void __fastunpack26(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<26>::Unpack(in, out);
}

void __fastunpack27(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<27>::Unpack(in, out);
}

void __fastunpack28(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<28>::Unpack(in, out);
}

void __fastunpack29(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<29>::Unpack(in, out);
}

void __fastunpack30(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<30>::Unpack(in, out);
}

void __fastunpack31(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<31>::Unpack(in, out);
}

void __fastunpack32(const uint32_t *__restrict in, uint64_t *__restrict out) {
	for (int k = 0; k < 32; ++k)
		out[k] = in[k];
}

void __fastunpack33(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<33>::Unpack(in, out);
}

void __fastunpack34(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<34>::Unpack(in, out);
}

void __fastunpack35(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<35>::Unpack(in, out);
}

void __fastunpack36(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<36>::Unpack(in, out);
}

void __fastunpack37(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<37>::Unpack(in, out);
}

void __fastunpack38(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<38>::Unpack(in, out);
}

void __fastunpack39(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<39>::Unpack(in, out);
}

void __fastunpack40(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<40>::Unpack(in, out);
}

void __fastunpack41(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<41>::Unpack(in, out);
}

void __fastunpack42(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<42>::Unpack(in, out);
}

void __fastunpack43(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<43>::Unpack(in, out);
}

void __fastunpack44(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<44>::Unpack(in, out);
}

void __fastunpack45(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<45>::Unpack(in, out);
}

void __fastunpack46(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<46>::Unpack(in, out);
}

void __fastunpack47(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<47>::Unpack(in, out);
}

void __fastunpack48(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<48>::Unpack(in, out);
}

void __fastunpack49(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<49>::Unpack(in, out);
}

void __fastunpack50(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<50>::Unpack(in, out);
}

void __fastunpack51(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<51>::Unpack(in, out);
}

void __fastunpack52(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<52>::Unpack(in, out);
}

void __fastunpack53(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<53>::Unpack(in, out);
}

void __fastunpack54(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<54>::Unpack(in, out);
}

void __fastunpack55(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<55>::Unpack(in, out);
}

void __fastunpack56(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<56>::Unpack(in, out);
}

void __fastunpack57(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<57>::Unpack(in, out);
}

void __fastunpack58(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<58>::Unpack(in, out);
}

void __fastunpack59(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<59>::Unpack(in, out);
}

void __fastunpack60(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<60>::Unpack(in, out);
}

void __fastunpack61(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<61>::Unpack(in, out);
}

void __fastunpack62(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<62>::Unpack(in, out);
}

void __fastunpack63(const uint32_t *__restrict in, uint64_t *__restrict out) {
	Unroller<63>::Unpack(in, out);
}

void __fastunpack64(const uint32_t *__restrict in, uint64_t *__restrict out) {
	for (int k = 0; k < 32; ++k) {
		out[k] = in[k * 2];
		out[k] |= static_cast<uint64_t>(in[k * 2 + 1]) << 32;
	}
}

// fastpack for 8 bits

void __fastpack1(const uint8_t *__restrict in, uint8_t *__restrict out) {
	Unroller8<1>::Pack(in, out);
}

void __fastpack2(const uint8_t *__restrict in, uint8_t *__restrict out) {
	Unroller8<2>::Pack(in, out);
}

void __fastpack3(const uint8_t *__restrict in, uint8_t *__restrict out) {
	Unroller8<3>::Pack(in, out);
}

void __fastpack4(const uint8_t *__restrict in, uint8_t *__restrict out) {
	Unroller8<4>::Pack(in, out);
}

void __fastpack5(const uint8_t *__restrict in, uint8_t *__restrict out) {
	Unroller8<5>::Pack(in, out);
}

void __fastpack6(const uint8_t *__restrict in, uint8_t *__restrict out) {
	Unroller8<6>::Pack(in, out);
}

void __fastpack7(const uint8_t *__restrict in, uint8_t *__restrict out) {
	Unroller8<7>::Pack(in, out);
}

void __fastpack8(const uint8_t *__restrict in, uint8_t *__restrict out) {
	for (int k = 0; k < 8; ++k)
		out[k] = in[k];
}

// fastpack for 16 bits

void __fastpack1(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<1>::Pack(in, out);
}

void __fastpack2(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<2>::Pack(in, out);
}

void __fastpack3(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<3>::Pack(in, out);
}

void __fastpack4(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<4>::Pack(in, out);
}

void __fastpack5(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<5>::Pack(in, out);
}

void __fastpack6(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<6>::Pack(in, out);
}

void __fastpack7(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<7>::Pack(in, out);
}

void __fastpack8(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<8>::Pack(in, out);
}

void __fastpack9(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<9>::Pack(in, out);
}

void __fastpack10(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<10>::Pack(in, out);
}

void __fastpack11(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<11>::Pack(in, out);
}

void __fastpack12(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<12>::Pack(in, out);
}

void __fastpack13(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<13>::Pack(in, out);
}

void __fastpack14(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<14>::Pack(in, out);
}

void __fastpack15(const uint16_t *__restrict in, uint16_t *__restrict out) {
	Unroller16<15>::Pack(in, out);
}

void __fastpack16(const uint16_t *__restrict in, uint16_t *__restrict out) {
	for (int k = 0; k < 16; ++k)
		out[k] = in[k];
}


// fastpack for 32 bits

void __fastpack1(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<1>::Pack(in, out);
}

void __fastpack2(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<2>::Pack(in, out);
}

void __fastpack3(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<3>::Pack(in, out);
}

void __fastpack4(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<4>::Pack(in, out);
}

void __fastpack5(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<5>::Pack(in, out);
}

void __fastpack6(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<6>::Pack(in, out);
}

void __fastpack7(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<7>::Pack(in, out);
}

void __fastpack8(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<8>::Pack(in, out);
}

void __fastpack9(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<9>::Pack(in, out);
}

void __fastpack10(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<10>::Pack(in, out);
}

void __fastpack11(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<11>::Pack(in, out);
}

void __fastpack12(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<12>::Pack(in, out);
}

void __fastpack13(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<13>::Pack(in, out);
}

void __fastpack14(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<14>::Pack(in, out);
}

void __fastpack15(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<15>::Pack(in, out);
}

void __fastpack16(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<16>::Pack(in, out);
}

void __fastpack17(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<17>::Pack(in, out);
}

void __fastpack18(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<18>::Pack(in, out);
}

void __fastpack19(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<19>::Pack(in, out);
}

void __fastpack20(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<20>::Pack(in, out);
}

void __fastpack21(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<21>::Pack(in, out);
}

void __fastpack22(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<22>::Pack(in, out);
}

void __fastpack23(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<23>::Pack(in, out);
}

void __fastpack24(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<24>::Pack(in, out);
}

void __fastpack25(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<25>::Pack(in, out);
}

void __fastpack26(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<26>::Pack(in, out);
}

void __fastpack27(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<27>::Pack(in, out);
}

void __fastpack28(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<28>::Pack(in, out);
}

void __fastpack29(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<29>::Pack(in, out);
}

void __fastpack30(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<30>::Pack(in, out);
}

void __fastpack31(const uint32_t *__restrict in, uint32_t *__restrict out) {
	Unroller<31>::Pack(in, out);
}

void __fastpack32(const uint32_t *__restrict in, uint32_t *__restrict out) {
	for (int k = 0; k < 32; ++k)
		out[k] = in[k];
}

// fastpack for 64 bits

void __fastpack1(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<1>::Pack(in, out);
}

void __fastpack2(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<2>::Pack(in, out);
}

void __fastpack3(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<3>::Pack(in, out);
}

void __fastpack4(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<4>::Pack(in, out);
}

void __fastpack5(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<5>::Pack(in, out);
}

void __fastpack6(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<6>::Pack(in, out);
}

void __fastpack7(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<7>::Pack(in, out);
}

void __fastpack8(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<8>::Pack(in, out);
}

void __fastpack9(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<9>::Pack(in, out);
}

void __fastpack10(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<10>::Pack(in, out);
}

void __fastpack11(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<11>::Pack(in, out);
}

void __fastpack12(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<12>::Pack(in, out);
}

void __fastpack13(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<13>::Pack(in, out);
}

void __fastpack14(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<14>::Pack(in, out);
}

void __fastpack15(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<15>::Pack(in, out);
}

void __fastpack16(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<16>::Pack(in, out);
}

void __fastpack17(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<17>::Pack(in, out);
}

void __fastpack18(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<18>::Pack(in, out);
}

void __fastpack19(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<19>::Pack(in, out);
}

void __fastpack20(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<20>::Pack(in, out);
}

void __fastpack21(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<21>::Pack(in, out);
}

void __fastpack22(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<22>::Pack(in, out);
}

void __fastpack23(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<23>::Pack(in, out);
}

void __fastpack24(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<24>::Pack(in, out);
}

void __fastpack25(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<25>::Pack(in, out);
}

void __fastpack26(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<26>::Pack(in, out);
}

void __fastpack27(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<27>::Pack(in, out);
}

void __fastpack28(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<28>::Pack(in, out);
}

void __fastpack29(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<29>::Pack(in, out);
}

void __fastpack30(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<30>::Pack(in, out);
}

void __fastpack31(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<31>::Pack(in, out);
}

void __fastpack32(const uint64_t *__restrict in, uint32_t *__restrict out) {
	for (int k = 0; k < 32; ++k) {
		out[k] = static_cast<uint32_t>(in[k]);
	}
}

void __fastpack33(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<33>::Pack(in, out);
}

void __fastpack34(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<34>::Pack(in, out);
}

void __fastpack35(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<35>::Pack(in, out);
}

void __fastpack36(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<36>::Pack(in, out);
}

void __fastpack37(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<37>::Pack(in, out);
}

void __fastpack38(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<38>::Pack(in, out);
}

void __fastpack39(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<39>::Pack(in, out);
}

void __fastpack40(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<40>::Pack(in, out);
}

void __fastpack41(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<41>::Pack(in, out);
}

void __fastpack42(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<42>::Pack(in, out);
}

void __fastpack43(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<43>::Pack(in, out);
}

void __fastpack44(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<44>::Pack(in, out);
}

void __fastpack45(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<45>::Pack(in, out);
}

void __fastpack46(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<46>::Pack(in, out);
}

void __fastpack47(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<47>::Pack(in, out);
}

void __fastpack48(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<48>::Pack(in, out);
}

void __fastpack49(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<49>::Pack(in, out);
}

void __fastpack50(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<50>::Pack(in, out);
}

void __fastpack51(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<51>::Pack(in, out);
}

void __fastpack52(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<52>::Pack(in, out);
}

void __fastpack53(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<53>::Pack(in, out);
}

void __fastpack54(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<54>::Pack(in, out);
}

void __fastpack55(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<55>::Pack(in, out);
}

void __fastpack56(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<56>::Pack(in, out);
}

void __fastpack57(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<57>::Pack(in, out);
}

void __fastpack58(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<58>::Pack(in, out);
}

void __fastpack59(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<59>::Pack(in, out);
}

void __fastpack60(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<60>::Pack(in, out);
}

void __fastpack61(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<61>::Pack(in, out);
}

void __fastpack62(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<62>::Pack(in, out);
}

void __fastpack63(const uint64_t *__restrict in, uint32_t *__restrict out) {
	Unroller<63>::Pack(in, out);
}

void __fastpack64(const uint64_t *__restrict in, uint32_t *__restrict out) {
	for (int i = 0; i < 32; ++i) {
		out[2 * i] = static_cast<uint32_t>(in[i]);
		out[2 * i + 1] = in[i] >> 32;
	}
}
} // namespace internal
} // namespace duckdb_fastpforlib