First Commit

externals/nihstro/src/disassembler.cpp

@@ -0,0 +1,316 @@
+// Copyright 2014 Tony Wasserka
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//     * Redistributions of source code must retain the above copyright
+//       notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above copyright
+//       notice, this list of conditions and the following disclaimer in the
+//       documentation and/or other materials provided with the distribution.
+//     * Neither the name of the owner nor the names of its contributors may
+//       be used to endorse or promote products derived from this software
+//       without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <algorithm>
+#include <cassert>
+#include <cmath>
+#include <iostream>
+#include <iomanip>
+#include <fstream>
+#include <sstream>
+#include <vector>
+#include <map>
+#include <stdint.h>
+
+#include "nihstro/bit_field.h"
+#include "nihstro/shader_bytecode.h"
+#include "nihstro/parser_shbin.h"
+
+using namespace nihstro;
+
+struct float24 {
+    static float24 FromFloat32(float val) {
+        float24 ret;
+        ret.value = val;
+        return ret;
+    }
+
+    // 16 bit mantissa, 7 bit exponent, 1 bit sign
+    // TODO: No idea if this works as intended
+    static float24 FromRawFloat24(uint32_t hex) {
+        float24 ret;
+        if ((hex & 0xFFFFFF) == 0) {
+            ret.value = 0;
+        } else {
+            uint32_t mantissa = hex & 0xFFFF;
+            uint32_t exponent = (hex >> 16) & 0x7F;
+            uint32_t sign = hex >> 23;
+            ret.value = std::pow(2.0f, (float)exponent-63.0f) * (1.0f + mantissa * std::pow(2.0f, -16.f));
+            if (sign)
+                ret.value = -ret.value;
+        }
+        return ret;
+    }
+
+    // Not recommended for anything but logging
+    float ToFloat32() const {
+        return value;
+    }
+
+private:
+    // Stored as a regular float, merely for convenience
+    // TODO: Perform proper arithmetic on this!
+    float value;
+};
+
+int main(int argc, char *argv[])
+{
+    // TODO: Make this check portable!
+    if (argc < 2) {
+        std::cout << "Error: No filename given" << std::endl;
+        return 0;
+    }
+
+    ShbinParser parser;
+    try {
+        parser.ReadHeaders(argv[1]);
+
+        for (int i = 0; i < parser.GetDVLBHeader().num_programs; ++i) {
+            auto& filename = parser.GetFilename(i);
+            std::cout << "DVLE " << std::setw(3) << std::dec << i << ": "
+//                      << "offset: 0x" << std::hex << std::setfill('0') << std::setw(4) << offset << ", "
+                      << std::hex << std::setfill('0')
+                      << "byte offset to main: 0x" << std::hex << std::setw(8) << 4 * parser.GetDVLEHeader(i).main_offset_words << "\", "
+                      << "original filename \"" << filename << "\""
+                      << std::setfill(' ') << std::dec << std::endl;
+        }
+
+        std::cout << "Got " << parser.GetDVLBHeader().num_programs << " DVLE headers" << std::endl;
+
+        if (argc < 3) {
+            std::cout << "Error: No DVLE index given" << std::endl;
+            return 0;
+        }
+
+        uint32_t dvle_index = std::stoi(std::string(argv[2]));
+        parser.ReadDVLE(dvle_index);
+
+        auto& dvle_header = parser.dvle_headers[dvle_index];
+        for (int i = 0; i < parser.shader_info.constant_table.size(); ++i) {
+            auto& info = parser.shader_info.constant_table[i];
+
+            switch (info.type) {
+            case ConstantInfo::Float:
+                std::cout << "Constant register info:  " << GetRegisterName(RegisterType::FloatUniform) << info.regid.Value()
+                          << " = (" << float24::FromRawFloat24(info.f.x).ToFloat32() << ", " << float24::FromRawFloat24(info.f.y).ToFloat32()
+                          << ", " << float24::FromRawFloat24(info.f.z).ToFloat32() << ", " << float24::FromRawFloat24(info.f.w).ToFloat32() << ")"
+                          << "  (raw: 0x" << std::hex << std::setfill('0') << std::setw(8) << info.full_first_word
+                          << " 0x" << std::setw(8) << info.f.x << " 0x" << std::setw(8) << info.f.y
+                          << " 0x" << std::setw(8) << info.f.z << " 0x" << std::setw(8) << info.f.w << std::dec << std::setfill( ' ') << ")"
+                          << std::endl;
+                break;
+
+            case ConstantInfo::Int:
+                std::cout << "Constant register info:  " << GetRegisterName(RegisterType::IntUniform) << info.regid.Value()
+                          << " = (" << (int)info.i.x << ", " << (int)info.i.y
+                          << ", " << (int)info.i.z << ", " << (int)info.i.w << ")"
+                          << "  (raw: 0x" << std::hex << std::setfill('0') << std::setw(8) << info.full_first_word
+                          << std::dec << std::setfill( ' ') << ")"
+                          << std::endl;
+                break;
+
+            case ConstantInfo::Bool:
+                std::cout << "Constant register info:  " << GetRegisterName(RegisterType::BoolUniform) << info.regid.Value()
+                          << " = " << std::boolalpha << (bool)info.b
+                          << "  (raw: 0x" << std::hex << std::setfill('0') << std::setw(8) << info.full_first_word
+                          << std::dec << std::setfill( ' ') << ")"
+                          << std::endl;
+                break;
+
+            default:
+            {
+                std::stringstream str("Unknown constant type: ");
+                str << std::hex << info.type.Value();
+                throw str.str();
+            }
+            }
+        }
+
+        for (int i = 0; i < parser.shader_info.label_table.size(); ++i) {
+            const auto& label_info = parser.shader_info.label_table[i];
+            std::cout << "Found label \"" << parser.shader_info.labels[label_info.program_offset]
+                      << "\" at program offset 0x" << std::hex << 4 * label_info.program_offset
+                      << std::endl;
+        }
+
+        for (auto& info : parser.shader_info.output_register_info)
+            std::cout << "Output register info:  " << GetRegisterName(RegisterType::Output) << info.id.Value() << '.' << std::setw(4) << std::left << info.GetMask()
+                      << " as " << std::setw(8) << info.GetSemanticName()
+                      << " (" << std::hex << std::setw(16) << std::setfill('0') << (uint64_t)info.hex << std::setfill(' ') << ")" << std::endl;
+
+        if (!parser.shader_info.uniform_table.empty()) {
+            size_t max_uniform_name_length = std::max_element(parser.shader_info.uniform_table.begin(), parser.shader_info.uniform_table.end(),
+                                                              [](const UniformInfo& i1, UniformInfo& i2) { return i1.name.length() < i2.name.length(); }
+                                                             )->name.length();
+            for (auto& uniform_info : parser.shader_info.uniform_table) {
+                bool is_range = (uniform_info.basic.reg_start != uniform_info.basic.reg_end);
+
+                std::cout << "Found uniform symbol \"" << std::setw(max_uniform_name_length) << uniform_info.name
+                          << "\" for register" << (is_range ? "s " : "  ") << std::dec
+                          << GetRegisterName(uniform_info.basic.GetStartType()) << uniform_info.basic.GetStartIndex();
+                if (is_range)
+                    std::cout << "-" << GetRegisterName(uniform_info.basic.GetEndType()) << uniform_info.basic.GetEndIndex();
+
+                std::cout << std::endl;
+            }
+        }
+
+// TODO:
+//        std::cout << "Disassembling " << parser.GetDVLPHeader().binary_size_words << " bytes from offset "
+//                  << dvlp_offset << " + " << dvlp_header.binary_offset << " = " << main_offset << " (main at byte offset " << "0x" << std::hex << 4 * dvle_header.main_offset_words << ")" << std::endl;
+    } catch (const std::string& err) {
+        std::cout << "Exception while reading \"" << argv[1] << "\": " << err << std::endl;
+        return 1;
+    } catch (const std::ios_base::failure& except) {
+        std::cout << "Exception while reading \"" << argv[1] << "\": ios_base::failure \"" << except.what() << "\" (invalid shbin?)" << std::endl;
+        return 1;
+    } catch (const std::bad_alloc&) {
+        std::cout << "Exception while reading \"" << argv[1] << "\": bad_alloc (invalid shbin?)" << std::endl;
+        return 1;
+    }
+
+    const ShaderInfo& shader_info = parser.shader_info;
+    for (uint32_t word = 0; word < shader_info.code.size(); ++word) {
+        std::cout.flags(std::ios::left | std::ios::hex);
+        if (shader_info.HasLabel(word)) {
+            std::cout << std::setw(8) << std::right << std::setfill('0') << 4*word
+                      << " [--------] " << shader_info.GetLabel(word) << ":" << std::endl;
+        }
+
+        Instruction instr = shader_info.code[word];
+        OpCode opcode = instr.opcode.Value();
+
+        std::cout << std::setw(8) << std::right << std::setfill('0') << 4*word << " "
+                  << "[" << std::setw(8) << std::right << std::setfill('0') << instr.hex << "]     "
+                  << std::setw(7) << std::left << std::setfill(' ') << opcode.GetInfo().name;
+
+        const SwizzlePattern& swizzle = shader_info.swizzle_info[instr.common.operand_desc_id].pattern;
+
+        // TODO: Not sure if name lookup works properly, yet!
+
+        if (opcode.GetInfo().type == OpCode::Type::Arithmetic) {
+            bool src_reversed = 0 != (opcode.GetInfo().subtype & OpCode::Info::SrcInversed);
+            auto src1 = instr.common.GetSrc1(src_reversed);
+            auto src2 = instr.common.GetSrc2(src_reversed);
+            auto dest = instr.common.dest.Value();
+
+            std::string src1_relative_address;
+            if (!instr.common.AddressRegisterName().empty())
+                src1_relative_address = "[" + instr.common.AddressRegisterName() + "]";
+
+            if (opcode.GetInfo().subtype & OpCode::Info::Dest) {
+                std::cout << std::setw(4) << std::right << dest.GetName() << "." << swizzle.DestMaskToString() << "  ";
+            } else {
+                std::cout << "    ";
+            }
+
+            if (opcode.GetInfo().subtype & OpCode::Info::Src1) {
+                std::cout << std::setw(8) << std::right << ((swizzle.negate_src1 ? "-" : "") + src1.GetName()) + src1_relative_address << "." << swizzle.SelectorToString(false) << "  ";
+            } else {
+                std::cout << "           ";
+            }
+
+            if (opcode.GetInfo().subtype & OpCode::Info::CompareOps) {
+                std::cout << instr.common.compare_op.ToString(instr.common.compare_op.x) << " " << instr.common.compare_op.ToString(instr.common.compare_op.y) << " ";
+            } else {
+            }
+
+            if (opcode.GetInfo().subtype & OpCode::Info::Src2) {
+                std::cout << std::setw(4) << std::right << (swizzle.negate_src2 ? "-" : "") + src2.GetName() << "." << swizzle.SelectorToString(true) << "   ";
+            } else {
+                std::cout << "            ";
+            }
+
+            std::cout << std::setw(2) << instr.common.operand_desc_id.Value() << " addr:" << instr.common.address_register_index.Value()
+                      << ";      " << shader_info.LookupDestName(dest, swizzle) << " <- " << (swizzle.negate_src1 ? "-" : "") + shader_info.LookupSourceName(src1, instr.common.address_register_index);
+            if (opcode.GetInfo().subtype & OpCode::Info::Src2)
+                std::cout << ", " << (swizzle.negate_src2 ? "-" : "") + shader_info.LookupSourceName(src2, 0);
+
+            std::cout << std::endl;
+        } else if (opcode.GetInfo().type == OpCode::Type::Conditional) {
+            std::cout << "if ";
+
+            if (opcode.GetInfo().subtype & OpCode::Info::HasCondition) {
+                const char* ops[] = {
+                    " || ", " && ", "", ""
+                };
+                if (instr.flow_control.op != instr.flow_control.JustY)
+                    std::cout << ((!instr.flow_control.refx) ? "!" : " ") << "cc.x";
+
+                std::cout << ops[instr.flow_control.op];
+
+                if (instr.flow_control.op != instr.flow_control.JustX)
+                    std::cout << ((!instr.flow_control.refy) ? "!" : " ") << "cc.y";
+
+                std::cout << " ";
+            } else if (opcode.GetInfo().subtype & OpCode::Info::HasUniformIndex) {
+                std::cout << "b" << instr.flow_control.bool_uniform_id << " ";
+            }
+
+            uint32_t target_addr = instr.flow_control.dest_offset;
+            uint32_t target_addr_else = instr.flow_control.dest_offset;
+
+            if (opcode.GetInfo().subtype & OpCode::Info::HasAlternative) {
+                std::cout << "else jump to 0x" << std::setw(4) << std::right << std::setfill('0') << 4 * instr.flow_control.dest_offset
+                          << " aka \"" << shader_info.GetLabel(instr.flow_control.dest_offset) << "\"";
+            } else if (opcode.GetInfo().subtype & OpCode::Info::HasExplicitDest) {
+                std::cout << "jump to 0x" << std::setw(4) << std::right << std::setfill('0') << 4 * instr.flow_control.dest_offset
+                          << " aka \"" << shader_info.GetLabel(instr.flow_control.dest_offset) << "\"";
+            } else {
+                // TODO: Handle other cases
+            }
+
+            if (opcode.GetInfo().subtype & OpCode::Info::HasFinishPoint) {
+                std::cout << "(return on " << std::setw(4) << std::right << std::setfill('0') << 4 * instr.flow_control.dest_offset + 4 * instr.flow_control.num_instructions << "\")";
+            }
+
+            std::cout << std::endl;
+        } else {
+            std::cout << std::endl;
+        }
+    }
+
+    std::cout << std::endl << "Swizzle patterns:" << std::endl;
+
+    for (int i = 0; i < shader_info.swizzle_info.size(); ++i) {
+        const auto& info = shader_info.swizzle_info[i];
+        const auto& pattern = info.pattern;
+        std::cout << "(" << std::setw(3) << std::right << std::hex << i << ") " << std::setw(8) << std::setfill('0') << pattern.hex << ": " << pattern.dest_mask.Value() << "   " <<
+                     " " << (int)pattern.negate_src1 << "  " <<
+                     " " << (int)pattern.src1_selector_3.Value() << " " << (int)pattern.src1_selector_2.Value() <<
+                     " " << (int)pattern.src1_selector_1.Value() << " " << (int)pattern.src1_selector_0.Value() << "   " <<
+                     " " << (int)pattern.negate_src2 << "  " <<
+                     " " << (int)pattern.src2_selector_3.Value() << " " << (int)pattern.src2_selector_2.Value() <<
+                     " " << (int)pattern.src2_selector_1.Value() << " " << (int)pattern.src2_selector_0.Value() << "   " <<
+                     " " << (int)pattern.negate_src3 << "  " <<
+                     " " << (int)pattern.src3_selector_3.Value() << " " << (int)pattern.src3_selector_2.Value() <<
+                     " " << (int)pattern.src3_selector_1.Value() << " " << (int)pattern.src3_selector_0.Value() << "   " <<
+                     " " <<  std::setw(8) << std::setfill('0') << info.unknown << std::setfill(' ') << std::endl;
+    }
+
+    return 0;
+}

externals/nihstro/src/parser_assembly.cpp

@@ -0,0 +1,277 @@
+// Copyright 2014 Tony Wasserka
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//     * Redistributions of source code must retain the above copyright
+//       notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above copyright
+//       notice, this list of conditions and the following disclaimer in the
+//       documentation and/or other materials provided with the distribution.
+//     * Neither the name of the owner nor the names of its contributors may
+//       be used to endorse or promote products derived from this software
+//       without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+// Enable this for detailed XML overview of parser results
+// #define BOOST_SPIRIT_DEBUG
+
+#include <boost/fusion/include/adapt_struct.hpp>
+#include <boost/spirit/include/qi.hpp>
+
+#include "nihstro/parser_assembly.h"
+#include "nihstro/parser_assembly_private.h"
+
+#include "nihstro/shader_binary.h"
+#include "nihstro/shader_bytecode.h"
+
+namespace spirit = boost::spirit;
+namespace qi = boost::spirit::qi;
+namespace ascii = boost::spirit::qi::ascii;
+namespace phoenix = boost::phoenix;
+
+using spirit::_1;
+using spirit::_2;
+using spirit::_3;
+using spirit::_4;
+
+using namespace nihstro;
+
+// Adapt parser data structures for use with boost::spirit
+
+BOOST_FUSION_ADAPT_STRUCT(
+    SetEmitInstruction::Flags,
+    (boost::optional<bool>, primitive_flag)
+    (boost::optional<bool>, invert_flag)
+)
+
+BOOST_FUSION_ADAPT_STRUCT(
+    SetEmitInstruction,
+    (OpCode, opcode)
+    (unsigned, vertex_id)
+    (SetEmitInstruction::Flags, flags)
+)
+
+phoenix::function<ErrorHandler> error_handler;
+
+template<typename Iterator, bool require_end_of_line>
+TrivialOpParser<Iterator, require_end_of_line>::TrivialOpParser(const ParserContext& context)
+                : TrivialOpParser::base_type(trivial_instruction),
+                  common(context),
+                  opcodes_trivial(common.opcodes_trivial),
+                  opcodes_compare(common.opcodes_compare),
+                  opcodes_float(common.opcodes_float),
+                  opcodes_flowcontrol(common.opcodes_flowcontrol),
+                  end_of_statement(common.end_of_statement),
+                  diagnostics(common.diagnostics) {
+
+        // Setup rules
+        if (require_end_of_line) {
+            opcode = qi::no_case[qi::lexeme[opcodes_trivial >> &ascii::space]];
+            trivial_instruction = opcode > end_of_statement;
+        } else {
+            opcode = qi::no_case[qi::lexeme[opcodes_trivial | opcodes_compare | opcodes_float[0]
+                                            | opcodes_float[1] | opcodes_float[2] | opcodes_float[3]
+                                            | opcodes_flowcontrol[0] | opcodes_flowcontrol[1] >> &ascii::space]];
+            trivial_instruction = opcode;
+        }
+
+        // Error handling
+        BOOST_SPIRIT_DEBUG_NODE(opcode);
+        BOOST_SPIRIT_DEBUG_NODE(trivial_instruction);
+
+        qi::on_error<qi::fail>(trivial_instruction, error_handler(phoenix::ref(diagnostics), _1, _2, _3, _4));
+}
+
+template<typename Iterator>
+SetEmitParser<Iterator>::SetEmitParser(const ParserContext& context)
+                : SetEmitParser::base_type(setemit_instruction),
+                  common(context),
+                  opcodes_setemit(common.opcodes_setemit),
+                  end_of_statement(common.end_of_statement),
+                  diagnostics(common.diagnostics) {
+
+        // Setup rules
+
+        auto comma_rule = qi::lit(',');
+
+        opcode = qi::lexeme[qi::no_case[opcodes_setemit] >> &ascii::space];
+
+        vertex_id = qi::uint_;
+        prim_flag = qi::lit("prim") >> &(!ascii::alnum) >> qi::attr(true);
+        inv_flag = qi::lit("inv") >> &(!ascii::alnum) >> qi::attr(true);
+        flags = ((comma_rule >> prim_flag) ^ (comma_rule >> inv_flag));
+
+        setemit_instruction = ((opcode >> vertex_id) >> (flags | qi::attr(SetEmitInstruction::Flags{}))) > end_of_statement;
+
+        // Error handling
+        BOOST_SPIRIT_DEBUG_NODE(opcode);
+        BOOST_SPIRIT_DEBUG_NODE(vertex_id);
+        BOOST_SPIRIT_DEBUG_NODE(prim_flag);
+        BOOST_SPIRIT_DEBUG_NODE(inv_flag);
+        BOOST_SPIRIT_DEBUG_NODE(flags);
+        BOOST_SPIRIT_DEBUG_NODE(setemit_instruction);
+
+        qi::on_error<qi::fail>(setemit_instruction, error_handler(phoenix::ref(diagnostics), _1, _2, _3, _4));
+}
+
+template<typename Iterator>
+LabelParser<Iterator>::LabelParser(const ParserContext& context)
+                : LabelParser::base_type(label), common(context),
+                  end_of_statement(common.end_of_statement),
+                  identifier(common.identifier),
+                  diagnostics(common.diagnostics) {
+
+        label = identifier >> qi::lit(':') > end_of_statement;
+
+        BOOST_SPIRIT_DEBUG_NODE(label);
+
+        qi::on_error<qi::fail>(label, error_handler(phoenix::ref(diagnostics), _1, _2, _3, _4));
+}
+template struct LabelParser<ParserIterator>;
+
+
+struct Parser::ParserImpl {
+    using Iterator = SourceTreeIterator;
+
+    ParserImpl(const ParserContext& context) : label(context), plain_instruction(context),
+                                               simple_instruction(context), instruction(context),
+                                               compare(context), flow_control(context),
+                                               setemit(context), declaration(context) {
+    }
+
+    unsigned Skip(Iterator& begin, Iterator end) {
+        unsigned lines_skipped = 0;
+        do {
+            parse(begin, end, skipper);
+            lines_skipped++;
+        } while (boost::spirit::qi::parse(begin, end, boost::spirit::qi::eol));
+
+        return --lines_skipped;
+    }
+
+    void SkipSingleLine(Iterator& begin, Iterator end) {
+        qi::parse(begin, end, *(qi::char_ - (qi::eol | qi::eoi)) >> (qi::eol | qi::eoi));
+    }
+
+    bool ParseLabel(Iterator& begin, Iterator end, StatementLabel* content) {
+        assert(content != nullptr);
+
+        return phrase_parse(begin, end, label, skipper, *content);
+    }
+
+    bool ParseOpCode(Iterator& begin, Iterator end, OpCode* content) {
+        assert(content != nullptr);
+
+        return phrase_parse(begin, end, plain_instruction, skipper, *content);
+    }
+
+    bool ParseSimpleInstruction(Iterator& begin, Iterator end, OpCode* content) {
+        assert(content != nullptr);
+
+        return phrase_parse(begin, end, simple_instruction, skipper, *content);
+    }
+
+    bool ParseFloatOp(Iterator& begin, Iterator end, FloatOpInstruction* content) {
+        assert(content != nullptr);
+
+        return phrase_parse(begin, end, instruction, skipper, *content);
+    }
+
+    bool ParseCompare(Iterator& begin, Iterator end, CompareInstruction* content) {
+        assert(content != nullptr);
+
+        return phrase_parse(begin, end, compare, skipper, *content);
+    }
+
+    bool ParseFlowControl(Iterator& begin, Iterator end, FlowControlInstruction* content) {
+        assert(content != nullptr);
+
+        return phrase_parse(begin, end, flow_control, skipper, *content);
+    }
+
+    bool ParseSetEmit(Iterator& begin, Iterator end, SetEmitInstruction* content) {
+        assert(content != nullptr);
+
+        return phrase_parse(begin, end, setemit, skipper, *content);
+    }
+
+    bool ParseDeclaration(Iterator& begin, Iterator end, StatementDeclaration* content) {
+        assert(content != nullptr);
+
+        return phrase_parse(begin, end, declaration, skipper, *content);
+    }
+
+private:
+    AssemblySkipper<Iterator>   skipper;
+
+    LabelParser<Iterator>       label;
+    TrivialOpParser<Iterator, false> plain_instruction;
+    TrivialOpParser<Iterator, true>  simple_instruction;
+    FloatOpParser<Iterator>     instruction;
+    CompareParser<Iterator>     compare;
+    FlowControlParser<Iterator> flow_control;
+    SetEmitParser<Iterator> setemit;
+    DeclarationParser<Iterator> declaration;
+};
+
+
+
+Parser::Parser(const ParserContext& context) : impl(new ParserImpl(context)) {
+};
+
+Parser::~Parser() {
+}
+
+unsigned Parser::Skip(Iterator& begin, Iterator end) {
+    return impl->Skip(begin, end);
+}
+
+void Parser::SkipSingleLine(Iterator& begin, Iterator end) {
+    impl->SkipSingleLine(begin, end);
+}
+
+bool Parser::ParseLabel(Iterator& begin, Iterator end, StatementLabel* label) {
+    return impl->ParseLabel(begin, end, label);
+}
+
+bool Parser::ParseOpCode(Iterator& begin, Iterator end, OpCode* opcode) {
+    return impl->ParseOpCode(begin, end, opcode);
+}
+
+bool Parser::ParseSimpleInstruction(Iterator& begin, Iterator end, OpCode* opcode) {
+    return impl->ParseSimpleInstruction(begin, end, opcode);
+}
+
+bool Parser::ParseFloatOp(Iterator& begin, Iterator end, FloatOpInstruction* instruction) {
+    return impl->ParseFloatOp(begin, end, instruction);
+}
+
+bool Parser::ParseCompare(Iterator& begin, Iterator end, CompareInstruction* content) {
+    return impl->ParseCompare(begin, end, content);
+}
+
+bool Parser::ParseFlowControl(Iterator& begin, Iterator end, FlowControlInstruction* content) {
+    return impl->ParseFlowControl(begin, end, content);
+}
+
+bool Parser::ParseSetEmit(Iterator& begin, Iterator end, SetEmitInstruction* content) {
+    return impl->ParseSetEmit(begin, end, content);
+}
+
+bool Parser::ParseDeclaration(Iterator& begin, Iterator end, StatementDeclaration* declaration) {
+    return impl->ParseDeclaration(begin, end, declaration);
+}

externals/nihstro/src/parser_assembly/common.cpp

@@ -0,0 +1,185 @@
+// Copyright 2014 Tony Wasserka
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//     * Redistributions of source code must retain the above copyright
+//       notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above copyright
+//       notice, this list of conditions and the following disclaimer in the
+//       documentation and/or other materials provided with the distribution.
+//     * Neither the name of the owner nor the names of its contributors may
+//       be used to endorse or promote products derived from this software
+//       without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+// Enable this for detailed XML overview of parser results
+// #define BOOST_SPIRIT_DEBUG
+
+#include <boost/fusion/include/adapt_struct.hpp>
+#include <boost/spirit/include/qi.hpp>
+
+#include "nihstro/parser_assembly.h"
+#include "nihstro/parser_assembly_private.h"
+
+#include "nihstro/shader_binary.h"
+#include "nihstro/shader_bytecode.h"
+
+namespace spirit = boost::spirit;
+namespace qi = boost::spirit::qi;
+namespace ascii = boost::spirit::qi::ascii;
+namespace phoenix = boost::phoenix;
+
+using spirit::_1;
+using spirit::_2;
+using spirit::_3;
+using spirit::_4;
+
+using namespace nihstro;
+
+// Adapt parser data structures for use with boost::spirit
+
+BOOST_FUSION_ADAPT_STRUCT(
+    IntegerWithSign,
+    (int, sign)
+    (unsigned, value)
+)
+
+/**
+ * Implementation of transform_attribute from std::vector<InputSwizzlerMask::Component> to InputSwizzlerMask.
+ * This eases swizzle mask parsing a lot.
+ */
+namespace boost { namespace spirit { namespace traits {
+template<>
+struct transform_attribute<InputSwizzlerMask, std::vector<InputSwizzlerMask::Component>, qi::domain>
+{
+    using Exposed = InputSwizzlerMask;
+
+    using type = std::vector<InputSwizzlerMask::Component>;
+
+    static void post(Exposed& val, const type& attr) {
+        val.num_components = attr.size();
+        for (size_t i = 0; i < attr.size(); ++i)
+            val.components[i] = attr[i];
+    }
+
+    static type pre(Exposed& val) {
+        type vec;
+        for (int i = 0; i < val.num_components; ++i)
+            vec.push_back(val.components[i]);
+        return vec;
+    }
+
+    static void fail(Exposed&) { }
+};
+}}} // namespaces
+
+template<>
+CommonRules<ParserIterator>::CommonRules(const ParserContext& context) {
+    // Setup symbol table
+    opcodes_trivial.add
+               ( "nop",      OpCode::Id::NOP      )
+               ( "end",      OpCode::Id::END      )
+               ( "emit",     OpCode::Id::EMIT     )
+               ( "else",     OpCode::Id::ELSE     )
+               ( "endif",    OpCode::Id::ENDIF    )
+               ( "endloop",  OpCode::Id::ENDLOOP  );
+
+    opcodes_float[0].add
+               ( "mova",     OpCode::Id::MOVA     );
+
+    opcodes_float[1].add
+               ( "exp",      OpCode::Id::EX2      )
+               ( "log",      OpCode::Id::LG2      )
+               ( "lit",      OpCode::Id::LIT      )
+               ( "flr",      OpCode::Id::FLR      )
+               ( "rcp",      OpCode::Id::RCP      )
+               ( "rsq",      OpCode::Id::RSQ      )
+               ( "mov",      OpCode::Id::MOV      );
+    opcodes_float[2].add
+               ( "add",      OpCode::Id::ADD      )
+               ( "dp3",      OpCode::Id::DP3      )
+               ( "dp4",      OpCode::Id::DP4      )
+               ( "dph",      OpCode::Id::DPH      )
+               ( "dst",      OpCode::Id::DST      )
+               ( "mul",      OpCode::Id::MUL      )
+               ( "sge",      OpCode::Id::SGE      )
+               ( "slt",      OpCode::Id::SLT      )
+               ( "max",      OpCode::Id::MAX      )
+               ( "min",      OpCode::Id::MIN      );
+    opcodes_float[3].add
+               ( "mad",      OpCode::Id::MAD      );
+
+    opcodes_compare.add
+               ( "cmp",      OpCode::Id::CMP      );
+
+    opcodes_flowcontrol[0].add
+               ( "break",    OpCode::Id::BREAK    )
+               ( "breakc",   OpCode::Id::BREAKC   )
+               ( "if",       OpCode::Id::GEN_IF   )
+               ( "loop",     OpCode::Id::LOOP     );
+    opcodes_flowcontrol[1].add
+               ( "jmp",      OpCode::Id::GEN_JMP  )
+               ( "call",     OpCode::Id::GEN_CALL );
+
+    opcodes_setemit.add
+               ( "setemitraw", OpCode::Id::SETEMIT );
+
+        signs.add( "+", +1)
+                 ( "-", -1);
+
+        // TODO: Add rgba/stq masks
+        swizzlers.add
+                     ( "x",    InputSwizzlerMask::x )
+                     ( "y",    InputSwizzlerMask::y )
+                     ( "z",    InputSwizzlerMask::z )
+                     ( "w",    InputSwizzlerMask::w );
+
+        // TODO: Make sure this is followed by a space or *some* separator
+        // TODO: Use qi::repeat(1,4)(swizzlers) instead of Kleene [failed to work when I tried, so make this work!]
+        // TODO: Use qi::lexeme[swizzlers] [crashed when I tried, so make this work!]
+        swizzle_mask = qi::attr_cast<InputSwizzlerMask, std::vector<InputSwizzlerMask::Component>>(*swizzlers);
+
+        identifier = qi::lexeme[qi::char_("a-zA-Z_") >> *qi::char_("a-zA-Z0-9_")];
+        peek_identifier = &identifier;
+
+        uint_after_sign = qi::uint_; // TODO: NOT dot (or alphanum) after this to prevent floats..., TODO: overflows?
+        sign_with_uint = signs > uint_after_sign;
+        index_expression_first_term = (qi::attr(+1) >> qi::uint_) | (peek_identifier > identifier);
+        index_expression_following_terms = (qi::lit('+') >> peek_identifier > identifier) | sign_with_uint;
+        index_expression = (-index_expression_first_term)           // the first element has an optional sign
+                            >> (*index_expression_following_terms); // following elements have a mandatory sign
+
+        expression = ((-signs) > peek_identifier > identifier) >> (-(qi::lit('[') > index_expression > qi::lit(']'))) >> *(qi::lit('.') > swizzle_mask);
+
+        end_of_statement = qi::omit[qi::eol | qi::eoi];
+
+        // Error handling
+        BOOST_SPIRIT_DEBUG_NODE(identifier);
+        BOOST_SPIRIT_DEBUG_NODE(uint_after_sign);
+        BOOST_SPIRIT_DEBUG_NODE(index_expression);
+        BOOST_SPIRIT_DEBUG_NODE(peek_identifier);
+        BOOST_SPIRIT_DEBUG_NODE(expression);
+        BOOST_SPIRIT_DEBUG_NODE(swizzle_mask);
+        BOOST_SPIRIT_DEBUG_NODE(end_of_statement);
+
+        diagnostics.Add(swizzle_mask.name(), "Expected swizzle mask after period");
+        diagnostics.Add(peek_identifier.name(), "Expected identifier");
+        diagnostics.Add(uint_after_sign.name(), "Expected integer number after sign");
+        diagnostics.Add(index_expression.name(), "Expected index expression between '[' and ']'");
+        diagnostics.Add(expression.name(), "Expected expression of a known identifier");
+        diagnostics.Add(end_of_statement.name(), "Expected end of statement");
+}

externals/nihstro/src/parser_assembly/compare.cpp

@@ -0,0 +1,106 @@
+// Copyright 2014 Tony Wasserka
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//     * Redistributions of source code must retain the above copyright
+//       notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above copyright
+//       notice, this list of conditions and the following disclaimer in the
+//       documentation and/or other materials provided with the distribution.
+//     * Neither the name of the owner nor the names of its contributors may
+//       be used to endorse or promote products derived from this software
+//       without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+// Enable this for detailed XML overview of parser results
+// #define BOOST_SPIRIT_DEBUG
+
+#include <boost/fusion/include/adapt_struct.hpp>
+#include <boost/spirit/include/qi.hpp>
+
+#include "nihstro/parser_assembly.h"
+#include "nihstro/parser_assembly_private.h"
+
+#include "nihstro/shader_binary.h"
+#include "nihstro/shader_bytecode.h"
+
+namespace spirit = boost::spirit;
+namespace qi = boost::spirit::qi;
+namespace ascii = boost::spirit::qi::ascii;
+namespace phoenix = boost::phoenix;
+
+using spirit::_1;
+using spirit::_2;
+using spirit::_3;
+using spirit::_4;
+
+using namespace nihstro;
+
+// Adapt parser data structures for use with boost::spirit
+
+/*BOOST_FUSION_ADAPT_STRUCT(
+    IntegerWithSign,
+    (int, sign)
+    (unsigned, value)
+)
+*/
+BOOST_FUSION_ADAPT_STRUCT(
+    CompareInstruction,
+    (OpCode, opcode)
+    (std::vector<Expression>, arguments)
+    (std::vector<Instruction::Common::CompareOpType::Op>, ops)
+)
+
+template<>
+CompareParser<ParserIterator>::CompareParser(const ParserContext& context)
+                : CompareParser::base_type(instruction),
+                  common(context),
+                  opcodes_compare(common.opcodes_compare),
+                  expression(common.expression),
+                  end_of_statement(common.end_of_statement),
+                  diagnostics(common.diagnostics) {
+
+        // TODO: Will this properly match >= ?
+        compare_ops.add
+                       ( "==", CompareOp::Equal )
+                       ( "!=", CompareOp::NotEqual )
+                       ( "<", CompareOp::LessThan )
+                       ( "<=", CompareOp::LessEqual )
+                       ( ">", CompareOp::GreaterThan )
+                       ( ">=", CompareOp::GreaterEqual );
+
+        // Setup rules
+
+        auto comma_rule = qi::lit(',');
+
+        opcode = qi::no_case[qi::lexeme[opcodes_compare >> &ascii::space]];
+        compare_op = qi::lexeme[compare_ops];
+
+        // cmp src1, src2, op1, op2
+        // TODO: Also allow "cmp src1 op1 src2, src1 op2 src2"
+        two_ops = compare_op > comma_rule > compare_op;
+        two_expressions = expression > comma_rule > expression;
+        instr[0] = opcode > two_expressions > comma_rule > two_ops;
+
+        instruction = instr[0] > end_of_statement;
+
+        // Error handling
+        BOOST_SPIRIT_DEBUG_NODE(instr[0]);
+        BOOST_SPIRIT_DEBUG_NODE(instruction);
+
+        qi::on_error<qi::fail>(instruction, error_handler(phoenix::ref(diagnostics), _1, _2, _3, _4));
+}

externals/nihstro/src/parser_assembly/declaration.cpp

@@ -0,0 +1,132 @@
+// Copyright 2014 Tony Wasserka
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//     * Redistributions of source code must retain the above copyright
+//       notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above copyright
+//       notice, this list of conditions and the following disclaimer in the
+//       documentation and/or other materials provided with the distribution.
+//     * Neither the name of the owner nor the names of its contributors may
+//       be used to endorse or promote products derived from this software
+//       without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+#include <boost/fusion/include/adapt_struct.hpp>
+#include <boost/spirit/include/qi.hpp>
+
+#include "nihstro/parser_assembly.h"
+#include "nihstro/parser_assembly_private.h"
+
+#include "nihstro/shader_binary.h"
+#include "nihstro/shader_bytecode.h"
+
+using spirit::_1;
+using spirit::_2;
+using spirit::_3;
+using spirit::_4;
+
+using namespace nihstro;
+
+// Adapt parser data structures for use with boost::spirit
+
+BOOST_FUSION_ADAPT_STRUCT(
+    ConditionInput,
+    (bool, invert)
+    (Identifier, identifier)
+    (boost::optional<InputSwizzlerMask>, swizzler_mask)
+)
+
+BOOST_FUSION_ADAPT_STRUCT(
+    StatementDeclaration::Extra,
+    (std::vector<float>, constant_value)
+    (boost::optional<OutputRegisterInfo::Type>, output_semantic)
+)
+
+BOOST_FUSION_ADAPT_STRUCT(
+    StatementDeclaration,
+    (std::string, alias_name)
+    (Identifier, identifier_start)
+    (boost::optional<Identifier>, identifier_end)
+    (boost::optional<InputSwizzlerMask>, swizzle_mask)
+    (StatementDeclaration::Extra, extra)
+)
+
+// Manually define a swap() overload for qi::hold to work.
+/*namespace boost {
+namespace spirit {
+void swap(nihstro::Condition& a, nihstro::Condition& b) {
+    boost::fusion::swap(a, b);
+}
+}
+}*/
+
+template<>
+DeclarationParser<ParserIterator>::DeclarationParser(const ParserContext& context)
+                : DeclarationParser::base_type(declaration),
+                  common(context),
+                  identifier(common.identifier), swizzle_mask(common.swizzle_mask),
+                  end_of_statement(common.end_of_statement),
+                  diagnostics(common.diagnostics) {
+
+        // Setup symbol table
+        output_semantics.add("position", OutputRegisterInfo::POSITION);
+        output_semantics.add("quaternion", OutputRegisterInfo::QUATERNION);
+        output_semantics.add("color", OutputRegisterInfo::COLOR);
+        output_semantics.add("texcoord0", OutputRegisterInfo::TEXCOORD0);
+        output_semantics.add("texcoord1", OutputRegisterInfo::TEXCOORD1);
+        output_semantics.add("texcoord2", OutputRegisterInfo::TEXCOORD2);
+        output_semantics.add("view", OutputRegisterInfo::VIEW);
+        output_semantics_rule = qi::lexeme[output_semantics];
+
+        // Setup rules
+
+        alias_identifier = qi::omit[qi::lexeme["alias" >> ascii::blank]] > identifier;
+
+        // e.g. 5.4 or (1.1, 2, 3)
+        constant = (qi::repeat(1)[qi::float_]
+                                  | (qi::lit('(') > (qi::float_ % qi::lit(',')) > qi::lit(')')));
+
+        dummy_const = qi::attr(std::vector<float>());
+        dummy_semantic = qi::attr(boost::optional<OutputRegisterInfo::Type>());
+
+        // match a constant or a semantic, and fill the respective other one with a dummy
+        const_or_semantic = (dummy_const >> output_semantics_rule) | (constant >> dummy_semantic);
+
+        // TODO: Would like to use +ascii::blank instead, but somehow that fails to parse lines like ".alias name o2.xy texcoord0" correctly
+        string_as = qi::omit[qi::no_skip[*/*+*/ascii::blank >> qi::lit("as") >> +ascii::blank]];
+
+        declaration = ((qi::lit('.') > alias_identifier) >> identifier >> -(qi::lit('-') > identifier) >> -(qi::lit('.') > swizzle_mask))
+                       >> (
+                            (string_as > const_or_semantic)
+                            | (dummy_const >> dummy_semantic)
+                          )
+                       > end_of_statement;
+
+        // Error handling
+        output_semantics_rule.name("output semantic after \"as\"");
+        alias_identifier.name("known preprocessor directive (i.e. alias).");
+        const_or_semantic.name("constant or semantic after \"as\"");
+
+        BOOST_SPIRIT_DEBUG_NODE(output_semantics_rule);
+        BOOST_SPIRIT_DEBUG_NODE(constant);
+        BOOST_SPIRIT_DEBUG_NODE(alias_identifier);
+        BOOST_SPIRIT_DEBUG_NODE(const_or_semantic);
+        BOOST_SPIRIT_DEBUG_NODE(declaration);
+
+        qi::on_error<qi::fail>(declaration, error_handler(phoenix::ref(diagnostics), _1, _2, _3, _4));
+}

externals/nihstro/src/parser_assembly/floatop.cpp

@@ -0,0 +1,116 @@
+// Copyright 2014 Tony Wasserka
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//     * Redistributions of source code must retain the above copyright
+//       notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above copyright
+//       notice, this list of conditions and the following disclaimer in the
+//       documentation and/or other materials provided with the distribution.
+//     * Neither the name of the owner nor the names of its contributors may
+//       be used to endorse or promote products derived from this software
+//       without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+// Enable this for detailed XML overview of parser results
+// #define BOOST_SPIRIT_DEBUG
+
+#include <boost/fusion/include/adapt_struct.hpp>
+#include <boost/spirit/include/qi.hpp>
+
+#include "nihstro/parser_assembly.h"
+#include "nihstro/parser_assembly_private.h"
+
+#include "nihstro/shader_binary.h"
+#include "nihstro/shader_bytecode.h"
+
+namespace spirit = boost::spirit;
+namespace qi = boost::spirit::qi;
+namespace ascii = boost::spirit::qi::ascii;
+namespace phoenix = boost::phoenix;
+
+using spirit::_1;
+using spirit::_2;
+using spirit::_3;
+using spirit::_4;
+
+using namespace nihstro;
+
+// Adapt parser data structures for use with boost::spirit
+
+BOOST_FUSION_ADAPT_STRUCT(
+    StatementInstruction,
+    (OpCode, opcode)
+    (std::vector<Expression>, expressions)
+)
+
+template<>
+FloatOpParser<ParserIterator>::FloatOpParser(const ParserContext& context)
+                : FloatOpParser::base_type(float_instruction),
+                  common(context),
+                  opcodes_float(common.opcodes_float),
+                  expression(common.expression),
+                  end_of_statement(common.end_of_statement),
+                  diagnostics(common.diagnostics) {
+
+        // Setup rules
+
+        auto comma_rule = qi::lit(',');
+
+        for (int i = 0; i < 4; ++i) {
+            // Make sure that a mnemonic is always followed by a space (such that e.g. "addbla" fails to match)
+            opcode[i] = qi::no_case[qi::lexeme[opcodes_float[i] >> &ascii::space]];
+        }
+
+        // chain of arguments for each group of opcodes
+        expression_chain[0] = expression;
+        for (int i = 1; i < 4; ++i) {
+            expression_chain[i] = expression_chain[i - 1] >> comma_rule > expression;
+        }
+
+        // e.g. "add o1, t2, t5"
+        float_instr[0] = opcode[0] > expression_chain[0];
+        float_instr[1] = opcode[1] > expression_chain[1];
+        float_instr[2] = opcode[2] > expression_chain[2];
+        float_instr[3] = opcode[3] > expression_chain[3];
+
+        float_instruction %= (float_instr[0] | float_instr[1] | float_instr[2] | float_instr[3]) > end_of_statement;
+
+        // Error handling
+        BOOST_SPIRIT_DEBUG_NODE(opcode[0]);
+        BOOST_SPIRIT_DEBUG_NODE(opcode[1]);
+        BOOST_SPIRIT_DEBUG_NODE(opcode[2]);
+        BOOST_SPIRIT_DEBUG_NODE(opcode[3]);
+
+        BOOST_SPIRIT_DEBUG_NODE(expression_chain[0]);
+        BOOST_SPIRIT_DEBUG_NODE(expression_chain[1]);
+        BOOST_SPIRIT_DEBUG_NODE(expression_chain[2]);
+        BOOST_SPIRIT_DEBUG_NODE(expression_chain[3]);
+
+        BOOST_SPIRIT_DEBUG_NODE(float_instr[0]);
+        BOOST_SPIRIT_DEBUG_NODE(float_instr[1]);
+        BOOST_SPIRIT_DEBUG_NODE(float_instr[2]);
+        BOOST_SPIRIT_DEBUG_NODE(float_instr[3]);
+        BOOST_SPIRIT_DEBUG_NODE(float_instruction);
+
+        diagnostics.Add(expression_chain[0].name(), "one argument");
+        diagnostics.Add(expression_chain[1].name(), "two arguments");
+        diagnostics.Add(expression_chain[2].name(), "three arguments");
+        diagnostics.Add(expression_chain[3].name(), "four arguments");
+
+        qi::on_error<qi::fail>(float_instruction, error_handler(phoenix::ref(diagnostics), _1, _2, _3, _4));
+}

externals/nihstro/src/parser_assembly/flowcontrol.cpp

@@ -0,0 +1,148 @@
+// Copyright 2014 Tony Wasserka
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//     * Redistributions of source code must retain the above copyright
+//       notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above copyright
+//       notice, this list of conditions and the following disclaimer in the
+//       documentation and/or other materials provided with the distribution.
+//     * Neither the name of the owner nor the names of its contributors may
+//       be used to endorse or promote products derived from this software
+//       without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+// Enable this for detailed XML overview of parser results
+// #define BOOST_SPIRIT_DEBUG
+
+#include <boost/fusion/include/adapt_struct.hpp>
+#include <boost/fusion/include/swap.hpp>
+#include <boost/spirit/include/qi.hpp>
+
+#include "nihstro/parser_assembly.h"
+#include "nihstro/parser_assembly_private.h"
+
+#include "nihstro/shader_binary.h"
+#include "nihstro/shader_bytecode.h"
+
+namespace spirit = boost::spirit;
+namespace qi = boost::spirit::qi;
+namespace ascii = boost::spirit::qi::ascii;
+namespace phoenix = boost::phoenix;
+
+using spirit::_1;
+using spirit::_2;
+using spirit::_3;
+using spirit::_4;
+
+using namespace nihstro;
+
+// Adapt parser data structures for use with boost::spirit
+
+BOOST_FUSION_ADAPT_STRUCT(
+    ConditionInput,
+    (bool, invert)
+    (Identifier, identifier)
+    (boost::optional<InputSwizzlerMask>, swizzler_mask)
+)
+
+BOOST_FUSION_ADAPT_STRUCT(
+    Condition,
+    (ConditionInput, input1)
+    (Instruction::FlowControlType::Op, op)
+    (ConditionInput, input2)
+)
+
+BOOST_FUSION_ADAPT_STRUCT(
+    FlowControlInstruction,
+    (OpCode, opcode)
+    (std::string, target_label)
+    (boost::optional<std::string>, return_label)
+    (boost::optional<Condition>, condition)
+)
+
+// Manually define a swap() overload for qi::hold to work.
+namespace boost {
+namespace spirit {
+void swap(nihstro::Condition& a, nihstro::Condition& b) {
+    boost::fusion::swap(a, b);
+}
+}
+}
+
+template<>
+FlowControlParser<ParserIterator>::FlowControlParser(const ParserContext& context)
+                : FlowControlParser::base_type(flow_control_instruction),
+                  common(context),
+                  opcodes_flowcontrol(common.opcodes_flowcontrol),
+                  expression(common.expression),
+                  identifier(common.identifier),
+                  swizzle_mask(common.swizzle_mask),
+                  end_of_statement(common.end_of_statement),
+                  diagnostics(common.diagnostics) {
+
+        condition_ops.add
+                   ( "&&",    ConditionOp::And     )
+                   ( "||",    ConditionOp::Or      );
+
+        // Setup rules
+
+        auto blank_rule = qi::omit[ascii::blank];
+        auto label_rule = identifier.alias();
+
+        opcode[0] = qi::lexeme[qi::no_case[opcodes_flowcontrol[0]] >> &ascii::space];
+        opcode[1] = qi::lexeme[qi::no_case[opcodes_flowcontrol[1]] >> &ascii::space];
+
+        condition_op = qi::lexeme[condition_ops];
+
+        negation = qi::matches[qi::lit("!")];
+
+        condition_input = negation >> identifier >> -(qi::lit('.') > swizzle_mask);
+
+        // May be a condition involving the conditional codes, or a reference to a uniform
+        // TODO: Make sure we use qi::hold wherever necessary
+        condition = qi::hold[condition_input >> condition_op >> condition_input]
+                    | (condition_input >> qi::attr(ConditionOp::JustX) >> qi::attr(ConditionInput{}));
+
+        // if condition
+        instr[0] = opcode[0]
+                   >> qi::attr("__dummy")  // Dummy label (set indirectly using else,endif, or endloop pseudo-instructions)
+                   >> qi::attr(boost::optional<std::string>()) // Dummy return label
+                   >> condition;
+
+        // call target_label until return_label if condition
+        instr[1] = opcode[1]
+                   >> label_rule
+                   >> -(qi::no_skip[(blank_rule >> qi::lit("until")) > blank_rule] >> label_rule)
+                   >> -(qi::no_skip[(blank_rule >> qi::lit("if")) > blank_rule] >> condition);
+
+        flow_control_instruction %= (instr[0] | instr[1]) > end_of_statement;
+
+        // Error handling
+        BOOST_SPIRIT_DEBUG_NODE(opcode[0]);
+        BOOST_SPIRIT_DEBUG_NODE(opcode[1]);
+        BOOST_SPIRIT_DEBUG_NODE(negation);
+        BOOST_SPIRIT_DEBUG_NODE(condition_op);
+        BOOST_SPIRIT_DEBUG_NODE(condition_input);
+        BOOST_SPIRIT_DEBUG_NODE(condition);
+
+        BOOST_SPIRIT_DEBUG_NODE(instr[0]);
+        BOOST_SPIRIT_DEBUG_NODE(instr[1]);
+        BOOST_SPIRIT_DEBUG_NODE(flow_control_instruction);
+
+        qi::on_error<qi::fail>(flow_control_instruction, error_handler(phoenix::ref(diagnostics), _1, _2, _3, _4));
+}

externals/nihstro/src/parser_shbin.cpp

@@ -0,0 +1,140 @@
+// Copyright 2014 Tony Wasserka
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//     * Redistributions of source code must retain the above copyright
+//       notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above copyright
+//       notice, this list of conditions and the following disclaimer in the
+//       documentation and/or other materials provided with the distribution.
+//     * Neither the name of the owner nor the names of its contributors may
+//       be used to endorse or promote products derived from this software
+//       without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "nihstro/parser_shbin.h"
+
+using namespace nihstro;
+
+void ShbinParser::ReadHeaders(const std::string& filename) {
+    file.exceptions(std::fstream::badbit | std::fstream::failbit | std::fstream::eofbit);
+    file.open(filename, std::fstream::in | std::fstream::binary);
+
+    file.seekg(0);
+    file.read((char*)&dvlb_header, sizeof(dvlb_header));
+    if (dvlb_header.magic_word != DVLBHeader::MAGIC_WORD) {
+        std::stringstream stream;
+        stream << "Wrong DVLB magic word: Got 0x" << std::hex << dvlb_header.magic_word;
+        throw stream.str();
+    }
+
+    dvle_offsets.resize(dvlb_header.num_programs);
+    dvle_headers.resize(dvlb_header.num_programs);
+    for (auto& offset : dvle_offsets) {
+        file.read((char*)&offset, sizeof(offset));
+    }
+
+    // DVLP comes directly after the DVLE offset table
+    dvlp_offset = file.tellg();
+    file.seekg(dvlp_offset);
+    file.read((char*)&dvlp_header, sizeof(dvlp_header));
+    if (dvlp_header.magic_word != DVLPHeader::MAGIC_WORD) {
+        std::stringstream stream;
+        stream << "Wrong DVLP magic word at offset " << std::hex << dvlp_offset << ": Got " << std::hex << dvlp_header.magic_word;
+        throw stream.str();
+    }
+
+    for (int i = 0; i < dvlb_header.num_programs; ++i) {
+        auto& dvle_header = dvle_headers[i];
+        file.seekg(dvle_offsets[i]);
+        file.read((char*)&dvle_header, sizeof(dvle_header));
+        if (dvle_header.magic_word != DVLEHeader::MAGIC_WORD) {
+            std::stringstream stream;
+            stream << "Wrong DVLE header in DVLE #" << i << ": " << std::hex << dvle_header.magic_word;
+            throw stream.str();
+        }
+    }
+
+    // TODO: Is there indeed exactly one filename per DVLE?
+    dvle_filenames.resize(dvlb_header.num_programs);
+    uint32_t offset = dvlp_offset + dvlp_header.filename_symbol_offset;
+    for (int i = 0; i < dvlb_header.num_programs; ++i) {
+        auto& filename = dvle_filenames[i];
+        filename = ReadSymbol(offset);
+        offset += filename.length() + 1;
+    }
+
+    // Read shader binary code
+    shader_info.code.resize(dvlp_header.binary_size_words);
+    file.seekg(dvlp_offset + dvlp_header.binary_offset);
+    file.read((char*)shader_info.code.data(), dvlp_header.binary_size_words * sizeof(Instruction));
+
+    // Read operand descriptor table
+    shader_info.swizzle_info.resize(dvlp_header.swizzle_info_num_entries);
+    file.seekg(dvlp_offset + dvlp_header.swizzle_info_offset);
+    file.read((char*)shader_info.swizzle_info.data(), dvlp_header.swizzle_info_num_entries * sizeof(SwizzleInfo));
+}
+
+void ShbinParser::ReadDVLE(int dvle_index) {
+    // TODO: Check if we have called ReadHeaders() before!
+
+    if (dvle_index >= dvlb_header.num_programs) {
+        std::stringstream stream;
+        stream << "Invalid DVLE index " << dvle_index << "given";
+        throw stream.str();
+    }
+
+    auto& dvle_header = dvle_headers[dvle_index];
+    auto& dvle_offset = dvle_offsets[dvle_index];
+
+    uint32_t symbol_table_offset = dvle_offset + dvle_header.symbol_table_offset;
+
+    shader_info.constant_table.resize(dvle_header.constant_table_size);
+    uint32_t constant_table_offset = dvle_offset + dvle_header.constant_table_offset;
+    file.seekg(constant_table_offset);
+    for (int i = 0; i < dvle_header.constant_table_size; ++i)
+        file.read((char*)&shader_info.constant_table[i], sizeof(ConstantInfo));
+
+    shader_info.label_table.resize(dvle_header.label_table_size);
+    uint32_t label_table_offset = dvle_offset + dvle_header.label_table_offset;
+    file.seekg(label_table_offset);
+    for (int i = 0; i < dvle_header.label_table_size; ++i)
+        file.read((char*)&shader_info.label_table[i], sizeof(LabelInfo));
+    for (const auto& label_info : shader_info.label_table)
+        shader_info.labels.insert({label_info.program_offset, ReadSymbol(symbol_table_offset + label_info.name_offset)});
+
+    shader_info.output_register_info.resize(dvle_header.output_register_table_size);
+    file.seekg(dvle_offset + dvle_header.output_register_table_offset);
+    for (auto& info : shader_info.output_register_info)
+        file.read((char*)&info, sizeof(OutputRegisterInfo));
+
+    shader_info.uniform_table.resize(dvle_header.uniform_table_size);
+    uint32_t uniform_table_offset = dvle_offset + dvle_header.uniform_table_offset;
+    file.seekg(uniform_table_offset);
+    for (int i = 0; i < dvle_header.uniform_table_size; ++i)
+        file.read((char*)&shader_info.uniform_table[i].basic, sizeof(shader_info.uniform_table[i].basic));
+    for (auto& uniform_info : shader_info.uniform_table)
+        uniform_info.name = ReadSymbol(symbol_table_offset + uniform_info.basic.symbol_offset);
+
+    main_offset = dvlp_offset + dvlp_header.binary_offset;
+}
+
+std::string ShbinParser::ReadSymbol(uint32_t offset) {
+    std::string name;
+    file.seekg(offset);
+    std::getline(file, name, '\0');
+    return name;
+};

externals/nihstro/src/preprocessor.cpp

@@ -0,0 +1,100 @@
+// Copyright 2015 Tony Wasserka
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//     * Redistributions of source code must retain the above copyright
+//       notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above copyright
+//       notice, this list of conditions and the following disclaimer in the
+//       documentation and/or other materials provided with the distribution.
+//     * Neither the name of the owner nor the names of its contributors may
+//       be used to endorse or promote products derived from this software
+//       without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <nihstro/parser_assembly_private.h>
+#include <nihstro/preprocessor.h>
+#include <nihstro/source_tree.h>
+
+#include <boost/spirit/include/qi.hpp>
+
+#include <fstream>
+
+namespace nihstro {
+
+template<typename Iterator>
+struct IncludeParser : qi::grammar<Iterator, std::string(), AssemblySkipper<Iterator>> {
+    using Skipper = AssemblySkipper<Iterator>;
+
+    IncludeParser() : IncludeParser::base_type(include) {
+        include = qi::lexeme[qi::lit(".include") >> &qi::ascii::space]
+                  > qi::lexeme[qi::lit("\"") > +qi::char_("a-zA-Z0-9./_\\-") > qi::lit("\"")]
+                  > qi::omit[qi::eol | qi::eoi];
+    }
+
+    qi::rule<Iterator, std::string(), Skipper> include;
+};
+
+
+SourceTree PreprocessAssemblyFile(const std::string& filename) {
+    SourceTree tree;
+    tree.file_info.filename = filename;
+
+    std::ifstream input_file(filename);
+    if (!input_file) {
+        throw std::runtime_error("Could not open input file " + filename);
+    }
+
+    std::string prefix;
+    {
+        auto last_slash = filename.find_last_of("/");
+        if (last_slash != std::string::npos)
+            prefix = filename.substr(0, last_slash + 1);
+    }
+
+    input_file.seekg(0, std::ios::end);
+    tree.code.resize(input_file.tellg());
+
+    input_file.seekg(0, std::ios::beg);
+    input_file.read(&tree.code[0], tree.code.size());
+    input_file.close();
+
+    auto cursor = tree.code.begin();
+
+    IncludeParser<decltype(cursor)> include_parser;
+    AssemblySkipper<decltype(cursor)> skipper;
+
+    while (cursor != tree.code.end()) {
+        std::string parsed_filename;
+        auto cursor_prev = cursor;
+        if (qi::phrase_parse(cursor, tree.code.end(), include_parser, skipper, parsed_filename)) {
+            if (parsed_filename[0] == '/')
+                throw std::runtime_error("Given filename must be relative to the path of the including file");
+
+            // TODO: Protect against circular inclusions
+            auto newtree = PreprocessAssemblyFile(prefix + parsed_filename);
+            tree.Attach(newtree, cursor_prev - tree.code.begin());
+            cursor = tree.code.erase(cursor_prev, cursor);
+            cursor = tree.code.insert(cursor, '\n');
+        } else {
+            // Skip this line
+            qi::parse(cursor, tree.code.end(), *(qi::char_ - (qi::eol | qi::eoi)) >> (qi::eol | qi::eoi));
+        }
+    }
+    return tree;
+}
+
+} // namespace

externals/nihstro/src/tests/parser.cpp

@@ -0,0 +1,393 @@
+// Copyright 2014 Tony Wasserka
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//     * Redistributions of source code must retain the above copyright
+//       notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above copyright
+//       notice, this list of conditions and the following disclaimer in the
+//       documentation and/or other materials provided with the distribution.
+//     * Neither the name of the owner nor the names of its contributors may
+//       be used to endorse or promote products derived from this software
+//       without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "nihstro/parser_assembly.h"
+#include "nihstro/parser_assembly_private.h"
+
+#include <boost/optional/optional_io.hpp>
+#include <boost/spirit/include/qi.hpp>
+
+#define BOOST_TEST_MODULE Parser
+#include <boost/test/unit_test.hpp>
+
+// Implement some ostream<< operators for BOOST_CHECK*
+namespace std {
+
+template<typename T>
+std::ostream& operator << (std::ostream& os, const std::vector<T>& vec) {
+    auto it = vec.begin();
+    os << "{";
+    if (!vec.empty())
+        os << " " << *it;
+    while (it != vec.end() && ++it != vec.end()) {
+        os << ", " << *it;
+    }
+    os << " }";
+    return os;
+}
+
+std::ostream& operator << (std::ostream& os, const nihstro::Expression& expr) {
+    if (expr.signed_identifier.sign)
+        os << *expr.signed_identifier.sign;
+    os << expr.signed_identifier.identifier;
+    if (expr.index)
+        os << "[" << *expr.index << "]";
+    for (auto& mask : expr.swizzle_masks)
+        os << "." << mask;
+    return os;
+}
+
+std::ostream& operator << (std::ostream& os, const nihstro::IndexExpression& expr) {
+    for (size_t i = 0; i < expr.GetCount(); ++i) {
+        if (i != 0)
+            os << ", ";
+
+        if (expr.IsRawIndex(i)) {
+            os << expr.GetRawIndex(i);
+        } else if (expr.IsAddressRegisterIdentifier(i)) {
+            os << expr.GetAddressRegisterIdentifier(i);
+        } else {
+            os << "?";
+        }
+    }
+    return os;
+}
+
+std::ostream& operator << (std::ostream& os, const nihstro::IntegerWithSign& num) {
+    os << num.GetValue();
+    return os;
+}
+
+std::ostream& operator << (std::ostream& os, const nihstro::Instruction::FlowControlType::Op& op) {
+    if (op == Instruction::FlowControlType::And)
+        os << "&&";
+    else if (op == Instruction::FlowControlType::Or)
+        os << "||";
+    else
+        os << "??";
+
+    return os;
+}
+
+std::ostream& operator << (std::ostream& os, const nihstro::ConditionInput& inp) {
+    if (inp.invert)
+        os << "!";
+
+    os << inp.identifier;
+    if (inp.swizzler_mask)
+        os << "." << *inp.swizzler_mask;
+
+    return os;
+}
+
+std::ostream& operator << (std::ostream& os, const nihstro::Condition& cond) {
+    os << "{ " << cond.input1;
+
+    if (cond.op != Instruction::FlowControlType::JustX)
+        os << " " << cond.op << " " << cond.input2;
+
+    os << " }";
+    return os;
+}
+
+} // namespace std
+
+// Utility comparison operators
+namespace nihstro {
+bool operator == (const IntegerWithSign& a, const IntegerWithSign& b) {
+    return a.sign == b.sign && a.value == b.value;
+}
+
+bool operator == (const Expression& a, const Expression& b) {
+    bool ret = true;
+    ret &= a.signed_identifier.sign == b.signed_identifier.sign;
+    ret &= a.signed_identifier.identifier == b.signed_identifier.identifier;
+    ret &= a.index == b.index;
+    ret &= a.swizzle_masks == b.swizzle_masks;
+    return ret;
+}
+
+bool operator == (const ConditionInput& a, const ConditionInput& b) {
+    return a.invert == b.invert && a.identifier == b.identifier && a.swizzler_mask == b.swizzler_mask;
+}
+
+bool operator == (const Condition& a, const Condition& b) {
+    return a.input1 == b.input1 && a.input2 == b.input2 && a.op == b.op;
+}
+
+} // namespace nihstro
+
+// Utility function to parse the given input. Upon match, returns the parsed contents, and returns no value upon failure.
+template<typename Attr, typename Parser, typename Skipper>
+static boost::optional<Attr> parse(const std::string& input, const Parser& parser, const Skipper& skipper) {
+    BOOST_TEST_MESSAGE(("Parsing \"" + input + "\"").c_str());
+    Attr attr;
+    SourceTree inp;
+    inp.code = input;
+    if (boost::spirit::qi::phrase_parse(inp.begin(), inp.end(), parser, skipper, attr))
+        return attr;
+    else
+        return {};
+}
+
+// Utility function to check that the given vectors are equal.
+template<typename T>
+static void CheckVector(const std::vector<T>& vec, const std::vector<T>& exp) {
+    // TODO: Could just check directly for equality, but need an ostream<< operator for that.
+    BOOST_CHECK_EQUAL(vec, exp);
+}
+
+// Utility function to convert a compile-time character to the corresponding swizzle mask component
+template<char a>
+static InputSwizzlerMask::Component MakeSwizzlerMaskComponent() {
+    static_assert(a == 'x' || a == 'y' || a == 'z' || a == 'w', "Invalid component");
+    if (a == 'x') return InputSwizzlerMask::x;
+    else if (a == 'y') return InputSwizzlerMask::y;
+    else if (a == 'z') return InputSwizzlerMask::z;
+    else return InputSwizzlerMask::w;
+}
+
+// Utility function to convert a series of up to four characters to the corresponding swizzle mask
+template<char a>
+static InputSwizzlerMask MakeInputSwizzlerMask() {
+    return { 1, { MakeSwizzlerMaskComponent<a>() } };
+}
+template<char a, char b>
+static InputSwizzlerMask MakeInputSwizzlerMask() {
+    return { 2, { MakeSwizzlerMaskComponent<a>(), MakeSwizzlerMaskComponent<b>() } };
+}
+template<char a, char b, char c>
+static InputSwizzlerMask MakeInputSwizzlerMask() {
+    return { 3, { MakeSwizzlerMaskComponent<a>(), MakeSwizzlerMaskComponent<b>(), MakeSwizzlerMaskComponent<c>() } };
+}
+template<char a, char b, char c, char d>
+static InputSwizzlerMask MakeInputSwizzlerMask() {
+    return { 4, { MakeSwizzlerMaskComponent<a>(), MakeSwizzlerMaskComponent<b>(), MakeSwizzlerMaskComponent<c>(), MakeSwizzlerMaskComponent<d>() } };
+}
+
+// Utility function to check that parsing of the first declaration statement was successfull and that the result matches the second declaration statements.
+static void CheckDeclaration(const boost::optional<StatementDeclaration>& declaration, const StatementDeclaration& exp) {
+    if (!declaration) {
+        BOOST_CHECK(false);
+    } else {
+        BOOST_CHECK_EQUAL(declaration->alias_name, exp.alias_name);
+        BOOST_CHECK_EQUAL(declaration->identifier_start, exp.identifier_start);
+        BOOST_CHECK_EQUAL(declaration->identifier_end, exp.identifier_end);
+        BOOST_CHECK_EQUAL(declaration->swizzle_mask, exp.swizzle_mask);
+        CheckVector(declaration->extra.constant_value, exp.extra.constant_value);
+        BOOST_CHECK_EQUAL(declaration->extra.output_semantic, exp.extra.output_semantic);
+    }
+}
+
+BOOST_AUTO_TEST_CASE(declaration) {
+    ParserContext context;
+    DeclarationParser<ParserIterator> declaration_parser(context);
+    AssemblySkipper<ParserIterator> skipper;
+
+    {
+    // Plain alias
+    auto declaration = parse<StatementDeclaration>(".alias my_alias r5.xy", declaration_parser, skipper);
+    CheckDeclaration(declaration, { "my_alias", "r5", {}, MakeInputSwizzlerMask<'x', 'y'>() });
+    }
+
+    {
+    // Array alias
+    auto declaration = parse<StatementDeclaration>(".alias my_alias r5-r10", declaration_parser, skipper);
+    CheckDeclaration(declaration, { "my_alias", "r5", std::string("r10") });
+    }
+
+    {
+    // Output alias
+    auto declaration = parse<StatementDeclaration>(".alias my_alias o5.xyz as texcoord0", declaration_parser, skipper);
+    CheckDeclaration(declaration, { "my_alias", "o5", {}, MakeInputSwizzlerMask<'x', 'y', 'z'>(), { {}, OutputRegisterInfo::TEXCOORD0 } });
+    }
+
+    {
+    // Output alias without output semantic
+    BOOST_CHECK_THROW(parse<StatementDeclaration>(".alias my_alias o5.xyz", declaration_parser, skipper),
+                      std::runtime_error);
+    }
+
+    {
+    // Constant alias
+    auto declaration = parse<StatementDeclaration>(".alias my_alias c5.xy as (1.0, -2.4)", declaration_parser, skipper);
+    CheckDeclaration(declaration, { "my_alias", "c5", {}, MakeInputSwizzlerMask<'x', 'y'>(), { { 1.0, -2.4 } } });
+    }
+}
+
+BOOST_AUTO_TEST_CASE(label) {
+    ParserContext context;
+    LabelParser<ParserIterator> label_parser(context);
+    AssemblySkipper<ParserIterator> skipper;
+
+    {
+    auto label = parse<StatementLabel>("my_label:", label_parser, skipper);
+    BOOST_CHECK(label);
+    BOOST_CHECK_EQUAL(*label, "my_label");
+    }
+}
+
+static void CheckParsedArithmeticInstruction(const boost::optional<FloatOpInstruction>& result, const FloatOpInstruction& exp) {
+    if (!result) {
+        BOOST_CHECK(false);
+    } else {
+        BOOST_CHECK_EQUAL(result->opcode, exp.opcode);
+        CheckVector(result->expressions, exp.expressions);
+    }
+}
+
+BOOST_AUTO_TEST_CASE(arithmetic) {
+    ParserContext context;
+
+    FloatOpParser<ParserIterator> arithmetic_parser(context);
+    AssemblySkipper<ParserIterator> skipper;
+
+    {
+    // one-argument instruction
+    auto result = parse<FloatOpInstruction>("mova r2.xy", arithmetic_parser, skipper);
+    Expression r2_xy = { { {}, "r2" }, {}, { MakeInputSwizzlerMask<'x', 'y'>() } };
+    CheckParsedArithmeticInstruction(result, { nihstro::OpCode::Id::MOVA, { r2_xy } });
+    }
+
+    {
+    // two-argument instruction
+    auto result = parse<FloatOpInstruction>("mov o0.wz, r2.xy", arithmetic_parser, skipper);
+    Expression o0_wz = { { {}, "o0" }, {}, { MakeInputSwizzlerMask<'w', 'z'>() } };
+    Expression r2_xy = { { {}, "r2" }, {}, { MakeInputSwizzlerMask<'x', 'y'>() } };
+    CheckParsedArithmeticInstruction(result, { nihstro::OpCode::Id::MOV, { o0_wz, r2_xy } });
+    }
+
+    {
+    // two-argument instruction with trivial register index
+    auto result = parse<FloatOpInstruction>("mov o0.wz, r2[5].xy", arithmetic_parser, skipper);
+    IndexExpression index_expr;
+    index_expr.emplace_back(IntegerWithSign{ +1, 5 });
+    Expression o0_wz = { { {}, "o0" }, {}, { MakeInputSwizzlerMask<'w', 'z'>() } };
+    Expression r2_xy = { { {}, "r2" }, index_expr, { MakeInputSwizzlerMask<'x', 'y'>() } };
+    CheckParsedArithmeticInstruction(result, { nihstro::OpCode::Id::MOV, { o0_wz, r2_xy } });
+    }
+
+    {
+    // two-argument instruction with nontrivial register index
+    auto result = parse<FloatOpInstruction>("mov o0.wz, r2[5+a1-4].xy", arithmetic_parser, skipper);
+    IndexExpression index_expr;
+    index_expr.emplace_back(IntegerWithSign{ +1, 5 });
+    index_expr.emplace_back("a1");
+    index_expr.emplace_back(IntegerWithSign{ -1, 4 });
+    Expression o0_wz = { { {}, "o0" }, {}, { MakeInputSwizzlerMask<'w', 'z'>() } };
+    Expression r2_xy = { { {}, "r2" }, index_expr, { MakeInputSwizzlerMask<'x', 'y'>() } };
+    CheckParsedArithmeticInstruction(result, { nihstro::OpCode::Id::MOV, { o0_wz, r2_xy } });
+    }
+
+    {
+    // three-argument instruction
+    auto result = parse<FloatOpInstruction>("add o0.xy, r2.xy, r3.xy", arithmetic_parser, skipper);
+    Expression o0_xy = { { {}, "o0" }, {}, { MakeInputSwizzlerMask<'x', 'y'>() } };
+    Expression r2_xy = { { {}, "r2" }, {}, { MakeInputSwizzlerMask<'x', 'y'>() } };
+    Expression r3_xy = { { {}, "r3" }, {}, { MakeInputSwizzlerMask<'x', 'y'>() } };
+    CheckParsedArithmeticInstruction(result, { nihstro::OpCode::Id::ADD, { o0_xy, r2_xy, r3_xy } });
+    }
+
+    {
+    // four-argument instruction
+    auto result = parse<FloatOpInstruction>("mad o0.xy, r2.xy, r3.xy, v4.xy", arithmetic_parser, skipper);
+    Expression o0_xy = { { {}, "o0" }, {}, { MakeInputSwizzlerMask<'x', 'y'>() } };
+    Expression r2_xy = { { {}, "r2" }, {}, { MakeInputSwizzlerMask<'x', 'y'>() } };
+    Expression r3_xy = { { {}, "r3" }, {}, { MakeInputSwizzlerMask<'x', 'y'>() } };
+    Expression v4_xy = { { {}, "v4" }, {}, { MakeInputSwizzlerMask<'x', 'y'>() } };
+    CheckParsedArithmeticInstruction(result, { nihstro::OpCode::Id::MAD, { o0_xy, r2_xy, r3_xy, v4_xy } });
+    }
+}
+
+static void CheckParsedFlowControlInstruction(const boost::optional<FlowControlInstruction>& result, const FlowControlInstruction& exp) {
+    if (!result) {
+        BOOST_CHECK(false);
+    } else {
+        BOOST_CHECK_EQUAL(result->opcode, exp.opcode);
+        BOOST_CHECK_EQUAL(result->target_label, exp.target_label);
+        BOOST_CHECK_EQUAL(result->return_label, exp.return_label);
+        BOOST_CHECK_EQUAL(result->condition, exp.condition);
+    }
+}
+
+BOOST_AUTO_TEST_CASE(flowcontrol) {
+    ParserContext context;
+    FlowControlParser<ParserIterator> parser(context);
+    AssemblySkipper<ParserIterator> skipper;
+
+    {
+    // If-conditionals with two arguments
+    auto result = parse<FlowControlInstruction>("if cc.x && !cc.y", parser, skipper);
+    Condition cond = { { false, "cc", MakeInputSwizzlerMask<'x'>() }, Instruction::FlowControlType::And, { true, "cc", MakeInputSwizzlerMask<'y'>() } };
+    CheckParsedFlowControlInstruction(result, { nihstro::OpCode::Id::GEN_IF, "__dummy", {}, cond } );
+    }
+
+    {
+    // If-conditionals with one argument with one component
+    auto result = parse<FlowControlInstruction>("if cc.x", parser, skipper);
+    Condition cond = { { false, "cc", MakeInputSwizzlerMask<'x'>() }, Instruction::FlowControlType::JustX };
+    CheckParsedFlowControlInstruction(result, { nihstro::OpCode::Id::GEN_IF, "__dummy", {}, cond } );
+    }
+
+    {
+    // If-conditionals with one argument with two components
+    auto result = parse<FlowControlInstruction>("if !cc.xy", parser, skipper);
+    Condition cond = { { true, "cc", MakeInputSwizzlerMask<'x', 'y'>() }, Instruction::FlowControlType::JustX };
+    CheckParsedFlowControlInstruction(result, { nihstro::OpCode::Id::GEN_IF, "__dummy", {}, cond });
+    }
+
+    {
+    // Loop instruction
+    auto result = parse<FlowControlInstruction>("loop i3.y", parser, skipper);
+    Condition cond = { { false, "i3", MakeInputSwizzlerMask<'y'>() }, Instruction::FlowControlType::JustX };
+    CheckParsedFlowControlInstruction(result, { OpCode::Id::LOOP, "__dummy", {}, cond });
+    }
+}
+
+static void CheckParsedCompareInstruction(const boost::optional<CompareInstruction>& result, const CompareInstruction& exp) {
+    if (!result) {
+        BOOST_CHECK(false);
+    } else {
+        BOOST_CHECK_EQUAL(result->opcode, exp.opcode);
+        CheckVector(result->arguments, exp.arguments);
+        CheckVector(result->ops, exp.ops);
+    }
+}
+
+BOOST_AUTO_TEST_CASE(compare) {
+    ParserContext context;
+    CompareParser<ParserIterator> parser(context);
+    AssemblySkipper<ParserIterator> skipper;
+
+    {
+    // Two separate comparisons
+    auto result = parse<CompareInstruction>("cmp r5.xy, r2.zx, ==, <=", parser, skipper);
+    Expression r5_xy = { { {}, "r5" }, {}, { MakeInputSwizzlerMask<'x', 'y'>() } };
+    Expression r2_zx = { { {}, "r2" }, {}, { MakeInputSwizzlerMask<'z', 'x'>() } };
+    CheckParsedCompareInstruction(result, { OpCode::Id::CMP, { r5_xy, r2_zx }, { Instruction::Common::CompareOpType::Equal, Instruction::Common::CompareOpType::LessEqual } });
+    }
+}

externals/nihstro/src/tests/source_tree_iterator.cpp

@@ -0,0 +1,169 @@
+// Copyright 2015 Tony Wasserka
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//     * Redistributions of source code must retain the above copyright
+//       notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above copyright
+//       notice, this list of conditions and the following disclaimer in the
+//       documentation and/or other materials provided with the distribution.
+//     * Neither the name of the owner nor the names of its contributors may
+//       be used to endorse or promote products derived from this software
+//       without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <iostream>
+#include "nihstro/source_tree.h"
+
+#define BOOST_TEST_MODULE SourceTreeIterator
+#include <boost/test/unit_test.hpp>
+
+#include <iterator>
+
+namespace std {
+
+std::ostream& operator << (std::ostream& os, const nihstro::SourceTree& tree) {
+    std::string::const_iterator it = tree.code.cbegin();
+    for (auto& child : tree.children) {
+        os << "\"";
+        os << std::string(it, tree.code.cbegin() + child.offset_within_parent);
+        os << "\"";
+        os << " { ";
+        os << child.tree;
+        os << " } ";
+        it = tree.code.cbegin() + child.offset_within_parent;
+    }
+    os << "\"" << std::string(it, tree.code.end()) << "\"";
+    return os;
+}
+
+}
+
+// Utility function to manually flatten the given tree into a string
+static std::string FlattenTree(const nihstro::SourceTree& tree) {
+    std::string ret;
+    std::string::const_iterator it = tree.code.cbegin();
+    for (auto& child : tree.children) {
+        ret += std::string(it, tree.code.cbegin() + child.offset_within_parent);
+        ret += FlattenTree(child.tree);
+        it = tree.code.cbegin() + child.offset_within_parent;
+    }
+    ret += std::string(it, tree.code.end());
+    return ret;
+}
+
+// Utility function to manually determine the size of the given tree
+static std::string::size_type TreeSize(const nihstro::SourceTree& tree) {
+    std::string::size_type ret = 0;
+    for (auto& child : tree.children) {
+        ret += TreeSize(child.tree);
+    }
+    ret += tree.code.length();
+    return ret;
+}
+
+#define CHECK_TREE(tree) do { \
+    /* Check length */ \
+    BOOST_CHECK_EQUAL(tree.end() - tree.begin(), TreeSize(tree)); \
+    BOOST_CHECK_EQUAL(std::distance(tree.begin(), tree.end()), TreeSize(tree)); \
+    /* Check forward iteration */ \
+    std::string flattened_tree; \
+    for (auto& val : tree) \
+        flattened_tree += val; \
+    auto reference_flattened_tree = FlattenTree(tree); \
+    BOOST_CHECK_EQUAL(flattened_tree, reference_flattened_tree); \
+    BOOST_CHECK_EQUAL_COLLECTIONS(flattened_tree.begin(), flattened_tree.end(), \
+                                  reference_flattened_tree.begin(), reference_flattened_tree.end()); \
+    \
+    /* Check reverse iteration */ \
+    flattened_tree.clear(); \
+    for (auto it = tree.end() - 1;; it -= 1) { \
+        flattened_tree += *it; \
+        if (it == tree.begin()) \
+            break; \
+    } \
+    std::reverse(reference_flattened_tree.begin(), reference_flattened_tree.end()); \
+    BOOST_CHECK_EQUAL(flattened_tree, reference_flattened_tree); \
+    BOOST_CHECK_EQUAL_COLLECTIONS(flattened_tree.begin(), flattened_tree.end(), \
+                                  reference_flattened_tree.begin(), reference_flattened_tree.end()); \
+ \
+} while (false)
+
+BOOST_AUTO_TEST_CASE(simple_tree) {
+    nihstro::SourceTree tree;
+
+    tree.code = "a b c";
+
+    CHECK_TREE(tree);
+}
+
+BOOST_AUTO_TEST_CASE(nested_tree) {
+    nihstro::SourceTree tree;
+    nihstro::SourceTree child1;
+    nihstro::SourceTree child2;
+
+    tree.code = "aXbXc";
+    child1.code = "child1";
+    child2.code = "child2";
+	tree.Attach(child1, 1).Attach(child2, 3);
+
+    CHECK_TREE(tree);
+}
+
+BOOST_AUTO_TEST_CASE(deep_tree) {
+    nihstro::SourceTree tree;
+    nihstro::SourceTree child1;
+    nihstro::SourceTree child1_child1;
+    nihstro::SourceTree child1_child2;
+    nihstro::SourceTree child1_child2_child1;
+    nihstro::SourceTree child1_child3;
+    nihstro::SourceTree child2;
+    nihstro::SourceTree child3;
+    nihstro::SourceTree child3_child1;
+    nihstro::SourceTree child4;
+
+    tree.code = "aaaXaaaXaaaXaaaXaaa";
+    child1.code = "FirstChild:bbbXbbbXbbbXbbb\n";
+    child1_child1.code = "FirstSubchildOfChild1:ccc";
+    child1_child2.code = "SecondSubchildOfChild1:dddXddd";
+    child1_child2_child1.code = "FirstSubsubchildOfSubchild2OfChild1:eee";
+    child1_child3.code = "ThirdSubchildOfChild1:fff";
+    child2.code = "SecondChild:ggg\n";
+    child3.code = "ThirdChild:hhhXhhh\n";
+    child3_child1.code = "FirstSubchildOfChild3:iii";
+    child4.code = "FourthChild:jjj\n";
+
+    child1_child2.Attach(child1_child2_child1, 26);
+    child1.Attach(child1_child1, 14).Attach(child1_child2, 18).Attach(child1_child3, 22);
+    child3.Attach(child3_child1, 14);
+    tree.Attach(child1, 3).Attach(child2, 7).Attach(child3, 11).Attach(child4, 15);
+
+    CHECK_TREE(tree);
+}
+
+BOOST_AUTO_TEST_CASE(subtree_at_begin_and_end) {
+    nihstro::SourceTree tree;
+    nihstro::SourceTree child1;
+    tree.code = "aaa";
+    child1.code = "bbb";
+
+    tree.Attach(child1, 0);
+    CHECK_TREE(tree);
+
+    tree.children.clear();
+    tree.Attach(child1, tree.code.length());
+    CHECK_TREE(tree);
+}