diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a245cb..495b54b 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,7 @@ set(SOURCE_FILES drm.c gstrtpreceiver.h # assembly/memcpymove-v7l.S + parse_x20_util.h ) include_directories("/usr/include/libdrm" "/usr/include/cairo" ) diff --git a/copy_util.h b/copy_util.h index 08636c5..b6aba32 100644 --- a/copy_util.h +++ b/copy_util.h @@ -10,7 +10,7 @@ #include #include - +//#define __ARM__ /*#include void memcpy_neon_8bytes(uint8_t* region2, const uint8_t* region1, size_t length){ @@ -47,6 +47,42 @@ void memcpy_neon_aligned(void* dst, const void * src, size_t length){ } }*/ +// From https://stackoverflow.com/questions/34888683/arm-neon-memcpy-optimized-for-uncached-memory +// and https://stackoverflow.com/questions/61210517/memcpy-for-arm-uncached-memory-for-arm64 +#ifdef __ARM__ +void my_copy(volatile void *dst, volatile const void *src, int sz){ + if (sz & 63) { + sz = (sz & -64) + 64; + } + asm volatile ("NEONCopyPLD: \n" + "sub %[dst], %[dst], #64 \n" + "1: \n" + "ldnp q0, q1, [%[src]] \n" + "ldnp q2, q3, [%[src], #32] \n" + "add %[dst], %[dst], #64 \n" + "subs %[sz], %[sz], #64 \n" + "add %[src], %[src], #64 \n" + "stnp q0, q1, [%[dst]] \n" + "stnp q2, q3, [%[dst], #32] \n" + "b.gt 1b \n" + : [dst]"+r"(dst), [src]"+r"(src), [sz]"+r"(sz) : : "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "cc", "memory"); +} +// https://wx.comake.online/doc/doc/SigmaStarDocs-SSC9341_Ispahan-ULS00V040-20210913/customer/faq/i6b0/system/i6b0/neon.html +// https://community.nxp.com/t5/i-MX-Processors/iMX6-EIM-transfer-speed-with-using-NEON-vld-vst-instructions/m-p/312256 +/*void __attribute__ ((noinline)) memcpy_neon_pld(void *dest, const void *src, size_t n) +{ + asm( + "NEONCopyPLD:\n" + " pld [r1, #0xC0]\n" //预取数据 + " vldm r1!,{d0-d7}\n" //从参数一r0（src）加载8*8=64个单通道8位数据 + " vstm r0!,{d0-d7}\n" //存储在目的地址r1（dst）中，同样是64个8位单通道8位数据 + " subs r2,r2,#0x40\n" //循环跳转参数，每次减64，总共循环次数=row*col*4/64 + " bgt NEONCopyPLD\n" //以前这里是bge，有问题。现在改成bgt。 + ); +}*/ +#endif + + #ifdef __ARM__ extern "C"{ // The memcpymove-v7l.S impl @@ -74,7 +110,9 @@ struct memcpy_args_t { void* memcpy_data_function(void* args_uncast){ struct memcpy_args_t* args=(struct memcpy_args_t*)args_uncast; #ifdef __ARM__ - mempcpy(args->dst,args->src,args->len); + //mempcpy(args->dst,args->src,args->len); + my_copy(args->dst,args->src,args->len); + //memcpy_neon_pld(args->dst,args->src,args->len); #else memcpy(args->dst,args->src,args->len); #endif diff --git a/gstrtpreceiver.cpp b/gstrtpreceiver.cpp index 4bb260b..dd6aab7 100644 --- a/gstrtpreceiver.cpp +++ b/gstrtpreceiver.cpp @@ -50,7 +50,7 @@ namespace pipeline{ if(codec==VideoCodec::H264){ std::stringstream ss; ss<<"video/x-h264"; - ss<<", stream-format=\"byte-stream\",alignment=au"; + ss<<", stream-format=\"byte-stream\",alignment=nal"; //ss<<", alignment=\"nal\""; ss<<" ! "; return ss.str(); diff --git a/main.cpp b/main.cpp index d1a6da7..6f5def8 100644 --- a/main.cpp +++ b/main.cpp @@ -49,6 +49,7 @@ extern "C" { #ifdef __cplusplus #include "gstrtpreceiver.h" #include "SchedulingHelper.hpp" +#include "parse_x20_util.h" #endif // This buffer size has no effect on the latency - @@ -96,7 +97,8 @@ int video_zpos = 1; int develop_rendering_mode=0; bool decode_h265=false; int gst_udp_port=-1; -bool x20_apply_fixes=false; +bool x20_force=false; +bool x20_auto=false; struct TSAccumulator m_decoding_latency; // NOTE: Does not track latency to end completely struct TSAccumulator m_decode_and_handover_display_latency; @@ -153,7 +155,7 @@ void map_copy_unmap(int fd_src,int fd_dst,int memory_size){ uint64_t before_memcpy=get_time_ms(); //memcpy_threaded(test_buffer,src_p,memory_size,3); //memcpy_threaded(dst_p,test_buffer,memory_size,3); - memcpy_threaded(dst_p,src_p,memory_size,3); + memcpy_threaded(dst_p,src_p,memory_size,2); end_sync(fd_src,false); end_sync(fd_dst,true); @@ -879,7 +881,6 @@ bool feed_packet_to_decoder(MppPacket *packet,void* data_p,int data_len){ decoder_stalled_count++; printf("Cannot feed decoder, stalled %d ?\n",decoder_stalled_count); return false; - break; } usleep(2 * 1000); } @@ -887,7 +888,8 @@ bool feed_packet_to_decoder(MppPacket *packet,void* data_p,int data_len){ } void configure_x20(MppPacket *packet){ - FILE *fp = fopen("/usr/local/bin/Header.h264", "rb"); + printf("Applying x20 hack\n"); + FILE *fp = fopen("/usr/local/bin/x20_header.h264", "rb"); assert(fp); fseek(fp, 0L, SEEK_END); long size = ftell(fp); @@ -900,6 +902,8 @@ void configure_x20(MppPacket *packet){ feed_packet_to_decoder(packet,tmp_data.data(),size); } +uint64_t first_frame_ms=0; +bool air_unit_discovery_finished= false; void read_gstreamerpipe_stream(MppPacket *packet){ assert(gst_udp_port!=-1); GstRtpReceiver receiver{gst_udp_port,decode_h265 ? 1 : 0}; @@ -911,10 +915,40 @@ void read_gstreamerpipe_stream(MppPacket *packet){ SchedulingHelper::set_thread_params_max_realtime("DisplayThread",SchedulingHelper::PRIORITY_REALTIME_LOW); first= false; } + if(!x20_force && x20_auto){ + // X20 auto detection + if(!air_unit_discovery_finished){ + const int x20_check=check_for_x20(frame->data(),frame->size()); + if(x20_check==1){ + // We have an x20 + configure_x20(packet); + air_unit_discovery_finished= true; + }else if(x20_check==2){ + // We have no x20 (definitely) + air_unit_discovery_finished= true; + }else{ + // Unknown if x20 or not + // As a bup, we assume no x20 after X seconds + if(first_frame_ms==0){ + first_frame_ms=get_time_ms(); + return ; + }else{ + const auto elapsed=get_time_ms()-first_frame_ms; + if(elapsed>5*1000){ + // Assume no x20 + printf("X20 or not unknown for > 5 seconds\n"); + air_unit_discovery_finished= true; + }else{ + // Skip this frame + return ; + } + } + } + } + } feed_packet_to_decoder(packet,frame->data(),frame->size()); }; - if(x20_apply_fixes){ - printf("Applying x20 hack\n"); + if(x20_force){ configure_x20(packet); } receiver.start_receiving(cb); @@ -1012,7 +1046,10 @@ void printHelp() { "\n" " --rmode - different rendering modes for development \n" "\n" - " --x20 - specific x20 fixe(s) \n" + " --x20-force - forces specific x20 fixe(s) (no autodetect), only works with x20\n" + "\n" + " --x20-auto - auto detect x20 or not as air, works with x20 AND rpi\n" + "\n" "\n", __DATE__ ); } @@ -1155,14 +1192,25 @@ int main(int argc, char **argv) develop_rendering_mode= atoi((char*)mode); continue; } - __OnArgument("--x20") { + __OnArgument("--x20-force") { + const char* mode = __ArgValue; + x20_force= true; + continue; + } + __OnArgument("--x20-auto") { const char* mode = __ArgValue; - x20_apply_fixes= true; + x20_auto= true; continue; } __EndParseConsoleArguments__ + // X20 force and x20 auto are exclusive + if(x20_auto && x20_force){ + printf("Cannot use x20 auto and force at the same time\n"); + assert(false); + } + if (enable_osd == 0 ) { video_zpos = 4; } diff --git a/nalu/CodecConfigFinder.hpp b/nalu/CodecConfigFinder.hpp new file mode 100644 index 0000000..beccca9 --- /dev/null +++ b/nalu/CodecConfigFinder.hpp @@ -0,0 +1,110 @@ +// +// Created by geier on 07/02/2020. +// + +#ifndef LIVEVIDEO10MS_KEYFRAMEFINDER_HPP +#define LIVEVIDEO10MS_KEYFRAMEFINDER_HPP + +#include +#include + +#include "NALU.hpp" +// #include +#include + +// Takes a continuous stream of NALUs and save SPS / PPS data +// For later use +class CodecConfigFinder { + private: + std::unique_ptr SPS = nullptr; + std::unique_ptr PPS = nullptr; + // VPS are only used in H265 + std::unique_ptr VPS = nullptr; + + public: + bool save_if_config(const NALU& nalu) { + if (nalu.getSize() <= 0) return false; + if (nalu.isSPS()) { + SPS = std::make_unique(nalu); + // qDebug()<<"SPS found"; + // qDebug()<(nalu); + // qDebug()<<"PPS found"; + return true; + } else if (nalu.IS_H265_PACKET && nalu.isVPS()) { + VPS = std::make_unique(nalu); + // qDebug()<<"VPS found"; + return true; + } + // qDebug()<<"not a keyframe"<<(int)nalu.getDataWithoutPrefix()[0]; + return false; + } + // H264 needs sps and pps + // H265 needs sps,pps and vps + bool all_config_available(const bool IS_H265 = false) { + if (IS_H265) { + return SPS != nullptr && PPS != nullptr && VPS != nullptr; + } + return SPS != nullptr && PPS != nullptr; + } + std::shared_ptr> get_config_data( + const bool IS_H265 = false) { + assert(all_config_available(IS_H265)); + if (IS_H265) { + // Looks like avcodec wants the VPS before sps and pps + auto& sps = SPS->get_nal(); + auto& pps = PPS->get_nal(); + auto& vps = VPS->get_nal(); + const auto size = sps.getSize() + pps.getSize() + vps.getSize(); + auto ret = std::make_unique>(size); + std::memcpy(ret->data(), vps.getData(), vps.getSize()); + auto offset = vps.getSize(); + std::memcpy(ret->data() + offset, sps.getData(), sps.getSize()); + offset += sps.getSize(); + std::memcpy(ret->data() + offset, pps.getData(), pps.getSize()); + return ret; + } + auto& sps = SPS->get_nal(); + auto& pps = PPS->get_nal(); + const auto size = sps.getSize() + pps.getSize(); + auto ret = std::make_shared>(size); + std::memcpy(ret->data(), sps.getData(), sps.getSize()); + std::memcpy(ret->data() + sps.getSize(), pps.getData(), pps.getSize()); + return ret; + } + // returns false if the config data (SPS,PPS,optional VPS) has changed + // true otherwise + bool check_is_still_same_config_data(const NALU& nalu) { + assert(all_config_available(nalu.IS_H265_PACKET)); + if (nalu.isSPS()) { + return compare(nalu, SPS->get_nal()); + } else if (nalu.isPPS()) { + return compare(nalu, PPS->get_nal()); + } else if (nalu.IS_H265_PACKET && nalu.isVPS()) { + return compare(nalu, VPS->get_nal()); + } + return true; + } + static void appendNaluData(std::vector& buff, const NALU& nalu) { + buff.insert(buff.begin(), nalu.getData(), nalu.getData() + nalu.getSize()); + } + void reset() { + SPS = nullptr; + PPS = nullptr; + VPS = nullptr; + } + const NALU& getCSD0() const { return SPS->get_nal(); } + const NALU& getCSD1() const { return PPS->get_nal(); } + const NALU& getVPS() const { return VPS->get_nal(); } + + public: + static bool compare(const NALU& n1, const NALU& n2) { + if (n1.getSize() != n2.getSize()) return false; + const int res = std::memcmp(n1.getData(), n2.getData(), n1.getSize()); + return res == 0; + } +}; + +#endif // LIVEVIDEO10MS_KEYFRAMEFINDER_HPP diff --git a/nalu/NALU.hpp b/nalu/NALU.hpp new file mode 100644 index 0000000..c7c95f9 --- /dev/null +++ b/nalu/NALU.hpp @@ -0,0 +1,237 @@ +// +// Created by Constantin on 2/6/2019. +// + +#ifndef LIVE_VIDEO_10MS_ANDROID_NALU_H +#define LIVE_VIDEO_10MS_ANDROID_NALU_H + +// https://github.com/Dash-Industry-Forum/Conformance-and-reference-source/blob/master/conformance/TSValidator/h264bitstream/h264_stream.h + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "NALUnitType.hpp" + +// dependency could be easily removed again + +static uint8_t extract_nal_unit_type(uint8_t value, bool is_h265) { + if (is_h265) { + return (value & 0x7E) >> 1; + } + return value & 0x1f; +} +static std::string x_get_nal_unit_type_as_string(int value, bool is_h265) { + if (value < 0) return "{<0}"; + const uint8_t extracted = extract_nal_unit_type(value, is_h265); + if (is_h265) { + return NALUnitType::H265::unit_type_to_string(extracted); + } + return NALUnitType::H264::unit_type_to_string(extracted); +} +static bool is_idr_frame(int value, bool is_h265) { + if (value <= 0) return false; + const uint8_t extracted = extract_nal_unit_type(value, is_h265); + if (is_h265) { + return extracted == NALUnitType::H265::NAL_UNIT_CODED_SLICE_IDR_N_LP; + } + return extracted == NALUnitType::H264::NAL_UNIT_TYPE_CODED_SLICE_IDR; +} + +/** + * NOTE: NALU only takes a c-style data pointer - it does not do any memory + * management. Use NALUBuffer if you need to store a NALU. Since H264 and H265 + * are that similar, we use this class for both (make sure to not call methds + * only supported on h265 with a h264 nalu,though) The constructor of the NALU + * does some really basic validation - make sure the parser never produces a + * NALU where this validation would fail + */ +class NALU { + public: + NALU(const uint8_t* data1, size_t data_len1, + const bool IS_H265_PACKET1 = false, + const std::chrono::steady_clock::time_point creationTime = + std::chrono::steady_clock::now()) + : m_data(data1), + m_data_len(data_len1), + IS_H265_PACKET(IS_H265_PACKET1), + creationTime{creationTime} { + assert(hasValidPrefix()); + assert(getSize() >= getMinimumNaluSize(IS_H265_PACKET1)); + m_nalu_prefix_size = get_nalu_prefix_size(); + } + ~NALU() = default; + // test video white iceland: Max 1024*117. Video might not be decodable if its + // NALU buffers size exceed the limit But a buffer size of 1MB accounts for + // 60fps video of up to 60MB/s or 480 Mbit/s. That should be plenty ! + static constexpr const auto NALU_MAXLEN = 1024 * 1024; + // Application should re-use NALU_BUFFER to avoid memory allocations + using NALU_BUFFER = std::array; + + private: + const uint8_t* m_data; + const size_t m_data_len; + int m_nalu_prefix_size; + + public: + const bool IS_H265_PACKET; + // creation time is used to measure latency + const std::chrono::steady_clock::time_point creationTime; + + public: + // returns true if starts with 0001, false otherwise + bool hasValidPrefixLong() const { + return m_data[0] == 0 && m_data[1] == 0 && m_data[2] == 0 && m_data[3] == 1; + } + // returns true if starts with 001 (short prefix), false otherwise + bool hasValidPrefixShort() const { + return m_data[0] == 0 && m_data[1] == 0 && m_data[2] == 1; + } + bool hasValidPrefix() const { + return hasValidPrefixLong() || hasValidPrefixShort(); + } + int get_nalu_prefix_size() const { + if (hasValidPrefixLong()) return 4; + return 3; + } + static std::size_t getMinimumNaluSize(const bool isH265) { + // 4 bytes prefix, 1 byte header for h264, 2 byte header for h265 + return isH265 ? 6 : 5; + } + + public: + // pointer to the NALU data with 0001 prefix + const uint8_t* getData() const { return m_data; } + // size of the NALU data with 0001 prefix + size_t getSize() const { return m_data_len; } + // pointer to the NALU data without 0001 prefix + const uint8_t* getDataWithoutPrefix() const { + return &getData()[m_nalu_prefix_size]; + } + // size of the NALU data without 0001 prefix + ssize_t getDataSizeWithoutPrefix() const { + return getSize() - m_nalu_prefix_size; + } + // return the nal unit type (quick) + int get_nal_unit_type() const { + if (IS_H265_PACKET) { + return (getDataWithoutPrefix()[0] & 0x7E) >> 1; + } + return getDataWithoutPrefix()[0] & 0x1f; + } + std::string get_nal_unit_type_as_string() const { + if (IS_H265_PACKET) { + return NALUnitType::H265::unit_type_to_string(get_nal_unit_type()); + } + return NALUnitType::H264::unit_type_to_string(get_nal_unit_type()); + } + + public: + bool isSPS() const { + if (IS_H265_PACKET) { + return get_nal_unit_type() == NALUnitType::H265::NAL_UNIT_SPS; + } + return (get_nal_unit_type() == NALUnitType::H264::NAL_UNIT_TYPE_SPS); + } + bool isPPS() const { + if (IS_H265_PACKET) { + return get_nal_unit_type() == NALUnitType::H265::NAL_UNIT_PPS; + } + return (get_nal_unit_type() == NALUnitType::H264::NAL_UNIT_TYPE_PPS); + } + // VPS NALUs are only possible in H265 + bool isVPS() const { + assert(IS_H265_PACKET); + return get_nal_unit_type() == NALUnitType::H265::NAL_UNIT_VPS; + } + bool is_aud() const { + if (IS_H265_PACKET) { + return get_nal_unit_type() == + NALUnitType::H265::NAL_UNIT_ACCESS_UNIT_DELIMITER; + } + return (get_nal_unit_type() == NALUnitType::H264::NAL_UNIT_TYPE_AUD); + } + bool is_sei() const { + if (IS_H265_PACKET) { + return get_nal_unit_type() == NALUnitType::H265::NAL_UNIT_PREFIX_SEI || + get_nal_unit_type() == NALUnitType::H265::NAL_UNIT_SUFFIX_SEI; + } + return (get_nal_unit_type() == NALUnitType::H264::NAL_UNIT_TYPE_SEI); + } + bool is_dps() const { + if (IS_H265_PACKET) { + // doesn't exist in h265 + return false; + } + return (get_nal_unit_type() == NALUnitType::H264::NAL_UNIT_TYPE_DPS); + } + bool is_config() const { + return isSPS() || isPPS() || (IS_H265_PACKET && isVPS()); + } + // keyframe / IDR frame + bool is_keyframe() const { + const auto nut = get_nal_unit_type(); + if (IS_H265_PACKET) { + return false; + } + if (nut == NALUnitType::H264::NAL_UNIT_TYPE_CODED_SLICE_IDR) { + return true; + } + return false; + } + bool is_frame_but_not_keyframe() const { + const auto nut = get_nal_unit_type(); + if (IS_H265_PACKET) return false; + return (nut == NALUnitType::H264::NAL_UNIT_TYPE_CODED_SLICE_NON_IDR); + } + static bool has_valid_prefix(const uint8_t* p) { + return p[0] == 0 && p[1] == 0 && p[2] == 0 && p[3] == 1 || + p[0] == 0 && p[1] == 0 && p[2] == 1; + } + static void write_prefix(uint8_t* p, bool long_prefix) { + if (long_prefix) { + p[0] = 0; + p[1] = 0; + p[2] = 0; + p[3] = 1; + } else { + p[0] = 0; + p[1] = 0; + p[2] = 1; + } + } +}; + +// Copies the nalu data into its own c++-style managed buffer. +class NALUBuffer { + public: + NALUBuffer(const uint8_t* data, int data_len, bool is_h265, + std::chrono::steady_clock::time_point creation_time) { + m_data = std::make_shared>(data, data + data_len); + m_nalu = std::make_unique(m_data->data(), m_data->size(), is_h265, + creation_time); + } + explicit NALUBuffer(const NALU& nalu) { + m_data = std::make_shared>( + nalu.getData(), nalu.getData() + nalu.getSize()); + m_nalu = std::make_shared(m_data->data(), m_data->size(), + nalu.IS_H265_PACKET, nalu.creationTime); + } + NALUBuffer(const NALUBuffer&) = delete; + NALUBuffer(const NALUBuffer&&) = delete; + + const NALU& get_nal() { return *m_nalu; } + + private: + std::shared_ptr> m_data; + std::shared_ptr m_nalu; +}; + +#endif // LIVE_VIDEO_10MS_ANDROID_NALU_H diff --git a/nalu/NALUnitType.hpp b/nalu/NALUnitType.hpp new file mode 100644 index 0000000..ce5b779 --- /dev/null +++ b/nalu/NALUnitType.hpp @@ -0,0 +1,397 @@ +// +// Created by consti10 on 11.11.20. +// + +#include + +#ifndef LIVEVIDEO10MS_NALUNITTYPE_H +#define LIVEVIDEO10MS_NALUNITTYPE_H + +// h264 types come from h264_stream +// h265 types are declared here +// This is just a lot of boilerplate code so I put it into its own .hpp file +namespace NALUnitType { + +namespace H264 { +enum { + // Table 7-1 NAL unit type codes + NAL_UNIT_TYPE_UNSPECIFIED = 0, // Unspecified + NAL_UNIT_TYPE_CODED_SLICE_NON_IDR = 1, // Coded slice of a non-IDR picture + NAL_UNIT_TYPE_CODED_SLICE_DATA_PARTITION_A = + 2, // Coded slice data partition A + NAL_UNIT_TYPE_CODED_SLICE_DATA_PARTITION_B = + 3, // Coded slice data partition B + NAL_UNIT_TYPE_CODED_SLICE_DATA_PARTITION_C = + 4, // Coded slice data partition C + NAL_UNIT_TYPE_CODED_SLICE_IDR = 5, // Coded slice of an IDR picture + NAL_UNIT_TYPE_SEI = 6, // Supplemental enhancement information (SEI) + NAL_UNIT_TYPE_SPS = 7, // Sequence parameter set + NAL_UNIT_TYPE_PPS = 8, // Picture parameter set + NAL_UNIT_TYPE_AUD = 9, // Access unit delimiter + NAL_UNIT_TYPE_END_OF_SEQUENCE = 10, // End of sequence + NAL_UNIT_TYPE_END_OF_STREAM = 11, // End of stream + NAL_UNIT_TYPE_FILLER = 12, // Filler data + NAL_UNIT_TYPE_SPS_EXT = 13, // Sequence parameter set extension + NAL_UNIT_TYPE_PREFIX_NAL = 14, // Prefix NAL unit + NAL_UNIT_TYPE_SUBSET_SPS = 15, // Subset Sequence parameter set + NAL_UNIT_TYPE_DPS = 16, // Depth Parameter Set + // 17..18 // Reserved + NAL_UNIT_TYPE_CODED_SLICE_AUX = + 19, // Coded slice of an auxiliary coded picture without partitioning + NAL_UNIT_TYPE_CODED_SLICE_SVC_EXTENSION = 20, // Coded slice of SVC extension + // 20..23 // Reserved + // 24..31 // Unspecified +}; +static std::string unit_type_to_string(const int nal_unit_type) { + std::string nal_unit_type_name; + switch (nal_unit_type) { + case NAL_UNIT_TYPE_UNSPECIFIED: + nal_unit_type_name = "NAL_UNIT_TYPE_UNSPECIFIED"; + break; // Unspecified + case NAL_UNIT_TYPE_CODED_SLICE_NON_IDR: + nal_unit_type_name = "NAL_UNIT_TYPE_CODED_SLICE_NON_IDR"; + break; // Coded slice of a non-IDR picture + case NAL_UNIT_TYPE_CODED_SLICE_DATA_PARTITION_A: + nal_unit_type_name = "NAL_UNIT_TYPE_CODED_SLICE_DATA_PARTITION_A"; + break; // Coded slice data partition A + case NAL_UNIT_TYPE_CODED_SLICE_DATA_PARTITION_B: + nal_unit_type_name = "NAL_UNIT_TYPE_CODED_SLICE_DATA_PARTITION_B"; + break; // Coded slice data partition B + case NAL_UNIT_TYPE_CODED_SLICE_DATA_PARTITION_C: + nal_unit_type_name = "NAL_UNIT_TYPE_CODED_SLICE_DATA_PARTITION_C"; + break; // Coded slice data partition C + case NAL_UNIT_TYPE_CODED_SLICE_IDR: + nal_unit_type_name = "NAL_UNIT_TYPE_CODED_SLICE_IDR"; + break; // Coded slice of an IDR picture + case NAL_UNIT_TYPE_SEI: + nal_unit_type_name = "NAL_UNIT_TYPE_SEI"; + break; // Supplemental enhancement information (SEI) + case NAL_UNIT_TYPE_SPS: + nal_unit_type_name = "NAL_UNIT_TYPE_SPS"; + break; // Sequence parameter set + case NAL_UNIT_TYPE_PPS: + nal_unit_type_name = "NAL_UNIT_TYPE_PPS"; + break; // Picture parameter set + case NAL_UNIT_TYPE_AUD: + nal_unit_type_name = "NAL_UNIT_TYPE_AUD"; + break; // Access unit delimiter + case NAL_UNIT_TYPE_END_OF_SEQUENCE: + nal_unit_type_name = "NAL_UNIT_TYPE_END_OF_SEQUENCE"; + break; // End of sequence + case NAL_UNIT_TYPE_END_OF_STREAM: + nal_unit_type_name = "NAL_UNIT_TYPE_END_OF_STREAM"; + break; // End of stream + case NAL_UNIT_TYPE_FILLER: + nal_unit_type_name = "NAL_UNIT_TYPE_FILLER"; + break; // Filler data + case NAL_UNIT_TYPE_SPS_EXT: + nal_unit_type_name = "SNAL_UNIT_TYPE_SPS_EXT"; + break; // Sequence parameter set extension + // 14..18 // Reserved + case NAL_UNIT_TYPE_CODED_SLICE_AUX: + nal_unit_type_name = "NAL_UNIT_TYPE_CODED_SLICE_AUX"; + break; + // Coded slice of an auxiliary coded picture without partitioning + // 20..23 // Reserved + // 24..31 // Unspecified + default: + nal_unit_type_name = + std::string("Unknown") + std::to_string(nal_unit_type); + break; + } + return nal_unit_type_name; +}; +} // namespace H264 + +namespace H265 { +enum NalUnitType { + NAL_UNIT_CODED_SLICE_TRAIL_N = 0, + NAL_UNIT_CODED_SLICE_TRAIL_R, + + NAL_UNIT_CODED_SLICE_TSA_N, + NAL_UNIT_CODED_SLICE_TSA_R, + + NAL_UNIT_CODED_SLICE_STSA_N, + NAL_UNIT_CODED_SLICE_STSA_R, + + NAL_UNIT_CODED_SLICE_RADL_N, + NAL_UNIT_CODED_SLICE_RADL_R, + + NAL_UNIT_CODED_SLICE_RASL_N, + NAL_UNIT_CODED_SLICE_RASL_R, + + NAL_UNIT_RESERVED_VCL_N10, + NAL_UNIT_RESERVED_VCL_R11, + NAL_UNIT_RESERVED_VCL_N12, + NAL_UNIT_RESERVED_VCL_R13, + NAL_UNIT_RESERVED_VCL_N14, + NAL_UNIT_RESERVED_VCL_R15, + + NAL_UNIT_CODED_SLICE_BLA_W_LP, + NAL_UNIT_CODED_SLICE_BLA_W_RADL, + NAL_UNIT_CODED_SLICE_BLA_N_LP, + NAL_UNIT_CODED_SLICE_IDR_W_RADL, + NAL_UNIT_CODED_SLICE_IDR_N_LP, + NAL_UNIT_CODED_SLICE_CRA, + NAL_UNIT_RESERVED_IRAP_VCL22, + NAL_UNIT_RESERVED_IRAP_VCL23, + + NAL_UNIT_RESERVED_VCL24, + NAL_UNIT_RESERVED_VCL25, + NAL_UNIT_RESERVED_VCL26, + NAL_UNIT_RESERVED_VCL27, + NAL_UNIT_RESERVED_VCL28, + NAL_UNIT_RESERVED_VCL29, + NAL_UNIT_RESERVED_VCL30, + NAL_UNIT_RESERVED_VCL31, + + NAL_UNIT_VPS, + NAL_UNIT_SPS, + NAL_UNIT_PPS, + NAL_UNIT_ACCESS_UNIT_DELIMITER, + NAL_UNIT_EOS, + NAL_UNIT_EOB, + NAL_UNIT_FILLER_DATA, + NAL_UNIT_PREFIX_SEI, + NAL_UNIT_SUFFIX_SEI, + NAL_UNIT_RESERVED_NVCL41, + NAL_UNIT_RESERVED_NVCL42, + NAL_UNIT_RESERVED_NVCL43, + NAL_UNIT_RESERVED_NVCL44, + NAL_UNIT_RESERVED_NVCL45, + NAL_UNIT_RESERVED_NVCL46, + NAL_UNIT_RESERVED_NVCL47, + NAL_UNIT_UNSPECIFIED_48, + NAL_UNIT_UNSPECIFIED_49, + NAL_UNIT_UNSPECIFIED_50, + NAL_UNIT_UNSPECIFIED_51, + NAL_UNIT_UNSPECIFIED_52, + NAL_UNIT_UNSPECIFIED_53, + NAL_UNIT_UNSPECIFIED_54, + NAL_UNIT_UNSPECIFIED_55, + NAL_UNIT_UNSPECIFIED_56, + NAL_UNIT_UNSPECIFIED_57, + NAL_UNIT_UNSPECIFIED_58, + NAL_UNIT_UNSPECIFIED_59, + NAL_UNIT_UNSPECIFIED_60, + NAL_UNIT_UNSPECIFIED_61, + NAL_UNIT_UNSPECIFIED_62, + NAL_UNIT_UNSPECIFIED_63, + NAL_UNIT_INVALID, +}; +static std::string unit_type_to_string(const int nal_unit_type) { + std::string nal_unit_type_name = "Unimpl"; + switch (nal_unit_type) { + case NalUnitType::NAL_UNIT_CODED_SLICE_TRAIL_N: + nal_unit_type_name = "NAL_UNIT_CODED_SLICE_TRAIL_N"; + break; + case NalUnitType::NAL_UNIT_CODED_SLICE_TRAIL_R: + nal_unit_type_name = "NAL_UNIT_CODED_SLICE_TRAIL_R"; + break; + + case NalUnitType::NAL_UNIT_CODED_SLICE_TSA_N: + nal_unit_type_name = "NAL_UNIT_CODED_SLICE_TSA_N"; + break; + case NalUnitType::NAL_UNIT_CODED_SLICE_TSA_R: + nal_unit_type_name = "NAL_UNIT_CODED_SLICE_TSA_R"; + break; + + case NalUnitType::NAL_UNIT_CODED_SLICE_STSA_N: + nal_unit_type_name = "NAL_UNIT_CODED_SLICE_STSA_N"; + break; + case NalUnitType::NAL_UNIT_CODED_SLICE_STSA_R: + nal_unit_type_name = "NAL_UNIT_CODED_SLICE_STSA_R"; + break; + + case NalUnitType::NAL_UNIT_CODED_SLICE_RADL_N: + nal_unit_type_name = "NAL_UNIT_CODED_SLICE_RADL_N"; + break; + case NalUnitType::NAL_UNIT_CODED_SLICE_RADL_R: + nal_unit_type_name = "NAL_UNIT_CODED_SLICE_RADL_R"; + break; + + case NalUnitType::NAL_UNIT_CODED_SLICE_RASL_N: + nal_unit_type_name = "NAL_UNIT_CODED_SLICE_RASL_N"; + break; + case NalUnitType::NAL_UNIT_CODED_SLICE_RASL_R: + nal_unit_type_name = "NAL_UNIT_CODED_SLICE_RASL_R"; + break; + + case NalUnitType::NAL_UNIT_RESERVED_VCL_N10: + nal_unit_type_name = "NAL_UNIT_RESERVED_VCL_N10"; + break; + case NalUnitType::NAL_UNIT_RESERVED_VCL_R11: + nal_unit_type_name = "NAL_UNIT_RESERVED_VCL_R11"; + break; + case NalUnitType::NAL_UNIT_RESERVED_VCL_N12: + nal_unit_type_name = "NAL_UNIT_RESERVED_VCL_N12"; + break; + case NalUnitType::NAL_UNIT_RESERVED_VCL_R13: + nal_unit_type_name = "NAL_UNIT_RESERVED_VCL_R13"; + break; + case NalUnitType::NAL_UNIT_RESERVED_VCL_N14: + nal_unit_type_name = "NAL_UNIT_RESERVED_VCL_N14"; + break; + case NalUnitType::NAL_UNIT_RESERVED_VCL_R15: + nal_unit_type_name = "NAL_UNIT_RESERVED_VCL_R15"; + break; + + case NalUnitType::NAL_UNIT_CODED_SLICE_BLA_W_LP: + nal_unit_type_name = "NAL_UNIT_CODED_SLICE_BLA_W_LP"; + break; + case NalUnitType::NAL_UNIT_CODED_SLICE_BLA_W_RADL: + nal_unit_type_name = "NAL_UNIT_CODED_SLICE_BLA_W_RADL"; + break; + case NalUnitType::NAL_UNIT_CODED_SLICE_BLA_N_LP: + nal_unit_type_name = "NAL_UNIT_CODED_SLICE_BLA_N_LP"; + break; + case NalUnitType::NAL_UNIT_CODED_SLICE_IDR_W_RADL: + nal_unit_type_name = "NAL_UNIT_CODED_SLICE_IDR_W_RADL"; + break; + case NalUnitType::NAL_UNIT_CODED_SLICE_IDR_N_LP: + nal_unit_type_name = "NAL_UNIT_CODED_SLICE_IDR_N_LP"; + break; + case NalUnitType::NAL_UNIT_CODED_SLICE_CRA: + nal_unit_type_name = "NAL_UNIT_CODED_SLICE_CRA"; + break; + case NalUnitType::NAL_UNIT_RESERVED_IRAP_VCL22: + nal_unit_type_name = "NAL_UNIT_RESERVED_IRAP_VCL22"; + break; + case NalUnitType::NAL_UNIT_RESERVED_IRAP_VCL23: + nal_unit_type_name = "NAL_UNIT_RESERVED_IRAP_VCL23"; + break; + + case NalUnitType::NAL_UNIT_RESERVED_VCL24: + nal_unit_type_name = "NAL_UNIT_RESERVED_VCL24"; + break; + case NalUnitType::NAL_UNIT_RESERVED_VCL25: + nal_unit_type_name = "NAL_UNIT_RESERVED_VCL25"; + break; + case NalUnitType::NAL_UNIT_RESERVED_VCL26: + nal_unit_type_name = "NAL_UNIT_RESERVED_VCL26"; + break; + case NalUnitType::NAL_UNIT_RESERVED_VCL27: + nal_unit_type_name = "NAL_UNIT_RESERVED_VCL27"; + break; + case NalUnitType::NAL_UNIT_RESERVED_VCL28: + nal_unit_type_name = "NAL_UNIT_RESERVED_VCL28"; + break; + case NalUnitType::NAL_UNIT_RESERVED_VCL29: + nal_unit_type_name = "NAL_UNIT_RESERVED_VCL29"; + break; + case NalUnitType::NAL_UNIT_RESERVED_VCL30: + nal_unit_type_name = "NAL_UNIT_RESERVED_VCL30"; + break; + case NalUnitType::NAL_UNIT_RESERVED_VCL31: + nal_unit_type_name = "NAL_UNIT_RESERVED_VCL31"; + break; + + case NalUnitType::NAL_UNIT_VPS: + nal_unit_type_name = "NAL_UNIT_VPS"; + break; + case NalUnitType::NAL_UNIT_SPS: + nal_unit_type_name = "NAL_UNIT_SPS"; + break; + case NalUnitType::NAL_UNIT_PPS: + nal_unit_type_name = "NAL_UNIT_PPS"; + break; + case NalUnitType::NAL_UNIT_ACCESS_UNIT_DELIMITER: + nal_unit_type_name = "NAL_UNIT_ACCESS_UNIT_DELIMITER"; + break; + case NalUnitType::NAL_UNIT_EOS: + nal_unit_type_name = "NAL_UNIT_EOS"; + break; + case NalUnitType::NAL_UNIT_EOB: + nal_unit_type_name = "NAL_UNIT_EOB"; + break; + case NalUnitType::NAL_UNIT_FILLER_DATA: + nal_unit_type_name = "NAL_UNIT_FILLER_DATA"; + break; + case NalUnitType::NAL_UNIT_PREFIX_SEI: + nal_unit_type_name = "NAL_UNIT_PREFIX_SEI"; + break; + case NalUnitType::NAL_UNIT_SUFFIX_SEI: + nal_unit_type_name = "NAL_UNIT_SUFFIX_SEI"; + break; + case NalUnitType::NAL_UNIT_RESERVED_NVCL41: + nal_unit_type_name = "NAL_UNIT_RESERVED_NVCL41"; + break; + case NalUnitType::NAL_UNIT_RESERVED_NVCL42: + nal_unit_type_name = "NAL_UNIT_RESERVED_NVCL42"; + break; + case NalUnitType::NAL_UNIT_RESERVED_NVCL43: + nal_unit_type_name = "NAL_UNIT_RESERVED_NVCL43"; + break; + case NalUnitType::NAL_UNIT_RESERVED_NVCL44: + nal_unit_type_name = "NAL_UNIT_RESERVED_NVCL44"; + break; + case NalUnitType::NAL_UNIT_RESERVED_NVCL45: + nal_unit_type_name = "NAL_UNIT_RESERVED_NVCL45"; + break; + case NalUnitType::NAL_UNIT_RESERVED_NVCL46: + nal_unit_type_name = "NAL_UNIT_RESERVED_NVCL46"; + break; + case NalUnitType::NAL_UNIT_RESERVED_NVCL47: + nal_unit_type_name = "NAL_UNIT_RESERVED_NVCL47"; + break; + case NalUnitType::NAL_UNIT_UNSPECIFIED_48: + nal_unit_type_name = "NAL_UNIT_UNSPECIFIED_48"; + break; + case NalUnitType::NAL_UNIT_UNSPECIFIED_49: + nal_unit_type_name = "NAL_UNIT_UNSPECIFIED_49"; + break; + case NalUnitType::NAL_UNIT_UNSPECIFIED_50: + nal_unit_type_name = "NAL_UNIT_UNSPECIFIED_50"; + break; + case NalUnitType::NAL_UNIT_UNSPECIFIED_51: + nal_unit_type_name = "NAL_UNIT_UNSPECIFIED_51"; + break; + case NalUnitType::NAL_UNIT_UNSPECIFIED_52: + nal_unit_type_name = "NAL_UNIT_UNSPECIFIED_52"; + break; + case NalUnitType::NAL_UNIT_UNSPECIFIED_53: + nal_unit_type_name = "NAL_UNIT_UNSPECIFIED_53"; + break; + case NalUnitType::NAL_UNIT_UNSPECIFIED_54: + nal_unit_type_name = "NAL_UNIT_UNSPECIFIED_54"; + break; + case NalUnitType::NAL_UNIT_UNSPECIFIED_55: + nal_unit_type_name = "NAL_UNIT_UNSPECIFIED_55"; + break; + case NalUnitType::NAL_UNIT_UNSPECIFIED_56: + nal_unit_type_name = "NAL_UNIT_UNSPECIFIED_56"; + break; + case NalUnitType::NAL_UNIT_UNSPECIFIED_57: + nal_unit_type_name = "NAL_UNIT_UNSPECIFIED_57"; + break; + case NalUnitType::NAL_UNIT_UNSPECIFIED_58: + nal_unit_type_name = "NAL_UNIT_UNSPECIFIED_58"; + break; + case NalUnitType::NAL_UNIT_UNSPECIFIED_59: + nal_unit_type_name = "NAL_UNIT_UNSPECIFIED_59"; + break; + case NalUnitType::NAL_UNIT_UNSPECIFIED_60: + nal_unit_type_name = "NAL_UNIT_UNSPECIFIED_60"; + break; + case NalUnitType::NAL_UNIT_UNSPECIFIED_61: + nal_unit_type_name = "NAL_UNIT_UNSPECIFIED_61"; + break; + case NalUnitType::NAL_UNIT_UNSPECIFIED_62: + nal_unit_type_name = "NAL_UNIT_UNSPECIFIED_62"; + break; + case NalUnitType::NAL_UNIT_UNSPECIFIED_63: + nal_unit_type_name = "NAL_UNIT_UNSPECIFIED_63"; + break; + case NalUnitType::NAL_UNIT_INVALID: + nal_unit_type_name = "NAL_UNIT_INVALID"; + break; + default: + nal_unit_type_name = + std::string("Unknown") + std::to_string(nal_unit_type); + break; + } + return nal_unit_type_name; +} +} // namespace H265 +} // namespace NALUnitType +#endif // LIVEVIDEO10MS_NALUNITTYPE_H diff --git a/nalu/fragment_helper.h b/nalu/fragment_helper.h new file mode 100644 index 0000000..712621b --- /dev/null +++ b/nalu/fragment_helper.h @@ -0,0 +1,48 @@ +// +// Created by consti10 on 07.01.24. +// + +#ifndef OPENHD_FRAGMENT_HELPER_H +#define OPENHD_FRAGMENT_HELPER_H + +#include + +static std::vector>> make_fragments( + const uint8_t* data, int data_len) { + std::vector>> fragments; + int bytes_used = 0; + const uint8_t* p = data; + static constexpr auto MAX_FRAGMENT_SIZE = 1024; + while (true) { + const int remaining = (int)data_len - bytes_used; + int len = 0; + if (remaining > MAX_FRAGMENT_SIZE) { + len = MAX_FRAGMENT_SIZE; + } else { + len = remaining; + } + std::shared_ptr> fragment = + std::make_shared>(p, p + len); + fragments.emplace_back(fragment); + p = p + len; + bytes_used += len; + if (bytes_used == data_len) { + break; + } + } + return fragments; +} + +static std::vector>> drop_fragments( + std::vector>> fragments) { + if (fragments.size() > 4) { + int random = std::rand(); + if (random % 8 == 0) { + // fragments.resize(fragments.size()/2); + fragments.resize(0); + } + } + return fragments; +} + +#endif // OPENHD_FRAGMENT_HELPER_H diff --git a/nalu/nalu_helper.h b/nalu/nalu_helper.h new file mode 100644 index 0000000..5bbf823 --- /dev/null +++ b/nalu/nalu_helper.h @@ -0,0 +1,54 @@ +// +// Created by consti10 on 27.12.23. +// + +#ifndef OPENHD_NALU_HELPER_H +#define OPENHD_NALU_HELPER_H + +#include + +static int find_next_nal(const uint8_t* data, int data_len) { + int nalu_search_state = 0; + for (int i = 0; i < data_len; i++) { + switch (nalu_search_state) { + case 0: + case 1: + if (data[i] == 0) + nalu_search_state++; + else + nalu_search_state = 0; + break; + case 2: + case 3: + if (data[i] == 0) { + nalu_search_state++; + } else if (data[i] == 1) { + // 0,0,0,1 or 0,0,1 + const int len = nalu_search_state == 2 ? 2 : 3; + if (i > len) { + return i - len; + } + nalu_search_state = 0; + } else { + nalu_search_state = 0; + } + break; + default: + break; + } + } + return data_len; +} + +static std::array EXAMPLE_AUD = {0, 0, 0, 1, 9, 48}; +static std::shared_ptr> get_h264_aud() { + return std::make_shared>( + EXAMPLE_AUD.data(), EXAMPLE_AUD.data() + EXAMPLE_AUD.size()); +} +static std::array EXAMPLE_START_CODE = {0, 0, 0, 1}; +static std::shared_ptr> get_h264_nalu_start_code() { + return std::make_shared>(EXAMPLE_START_CODE.begin(), + EXAMPLE_START_CODE.end()); +} + +#endif // OPENHD_NALU_HELPER_H diff --git a/other/Header.h264 b/other/x20_header.h264 similarity index 100% rename from other/Header.h264 rename to other/x20_header.h264 diff --git a/parse_x20_util.h b/parse_x20_util.h new file mode 100644 index 0000000..2891ab0 --- /dev/null +++ b/parse_x20_util.h @@ -0,0 +1,65 @@ +// +// Created by consti10 on 18.04.24. +// + +#ifndef FPVUE_PARSE_X20_UTIL_H +#define FPVUE_PARSE_X20_UTIL_H + + +#include + + +#include "nalu/NALU.hpp" + + +static void print_data(const uint8_t* data,int data_len){ + printf("[\n"); + for(int i=0;i