Trying to login to RDP using AS3

Apparently the majority of the buffer is little endian, but several bytes at its start are expected to be big endian numbers of 16 bit (short). This means, you have to write data in little endian as if it'll be interpreted as big endian. In order to convert the data from big endian to little endian, you can use a temporary ByteArray that has its endian set to big, write data in it, then call writeBytes() on your main buffer array, then clear the temporary big endian array. Writing constants can be done manually, since you can shift the byte order yourself, say when you are writing 0x0005 in big endian as short, you just do write 0x0500 as little endian instead. You seemingly wrote the code with the extraneous dataBuffer with endian being big, so you know this technique. Still, it's better that you will just produce a proper dataBuffer in the function. I'm attempting to fix your code below basing on the connectoid code I have downloaded, so that it will return a properly formed ByteArray with endian being little - this only matters if you'll read ordered data from it, not when you read bytes.

public function sendMcsData(): ByteArray {
    trace("Secure.sendMcsData");
    var num_channels: int = 2;
    var dataBuffer:ByteArray=new ByteArray(); //RdpPacket_Localised dataBuffer = new RdpPacket_Localised(512);
    // it's better to build the data buffer in the function, as in java, otherwise you can receive interference
    dataBuffer.endian=Endian.LITTLE_ENDIAN; // for clarity
    var hostlen: int = 2 * "myhost.ath.cx".length; // hardcoded? TODO FIX
    if (hostlen > 30) {
        hostlen = 30;
    }
    var length: int = 158;
    length += 76 + 12 + 4; // Options.use_rdp5 is true, apparently
    length += num_channels * 12 + 8;
    dataBuffer.writeShort(0x0500); // writing big-endian 0x5 *unknown*
    dataBuffer.writeShort(0x1400); // writing big-endian 0x14
    dataBuffer.writeByte(0x7c); //set 8 is write byte 
    //write short is setbigendian 16 //
    dataBuffer.writeShort(0x0100); // writing big-endian 0x01
    var be:ByteArray=new ByteArray();
    be.endian=Endian.BIG_ENDIAN; // create big-endian array for the data that's not static
    be.writeShort(length | 0x8000); // remaining length
    dataBuffer.writeBytes(be);
    be.clear(); // so that extra writing will not spoil the array
    dataBuffer.writeShort(0x0800); // writing big-endian 0x08 (length?)
    dataBuffer.writeShort(0x1000); // writing big-endian 16 (0x10)
    dataBuffer.writeByte(0);
    dataBuffer.writeShort(0xc001); // this one is little endian by default
    dataBuffer.writeByte(0);
    dataBuffer.writeUnsignedInt(0x61637544);
    //dataBuffer.setLittleEndian32(0x61637544); // "Duca" ?!
    be.writeShort((length - 14) | 0x8000); // remaining length
    dataBuffer.writeBytes(be);
    be.clear();
    dataBuffer.writeShort(SEC_TAG_CLI_INFO);
    dataBuffer.writeShort(212); // length
    dataBuffer.writeShort(4);
    dataBuffer.writeShort(8);
    dataBuffer.writeShort(600); // Options.width
    dataBuffer.writeShort(1024); // Options.height
    dataBuffer.writeShort(0xca01);
    dataBuffer.writeShort(0xaa03);
    dataBuffer.writeInt(0x0409); //Options.keylayout, default English/US - fixed
    dataBuffer.writeInt(2600); // or 0ece
    dataBuffer.writeBytes(b3);
    // // client
    // build? we
    // are 2600
    // compatible
    // :-)
    /* Unicode name of client, padded to 32 bytes */
    var targetPos:int=dataBuffer.position+32; // to account for padding
    dataBuffer.writeMultiByte("myhost.ath.cx".toLocaleUpperCase(), "UTF-16"); 
    // buffer.outUnicodeString(Options.hostname.toUpperCase(), hostlen);
    // apparently encoding is used "Unicode" that is UTF-16. If that will not work, set UTF-8 here
    // and by all means check what is on the wire when you connect via conventional RDP

    dataBuffer.position = targetPos;
    // this seems to be your mistake in converting position truncate, 
    // as position after writing already accounts for the writing been processed.
    // This line alone can be the source of size discrepancy you observe.
    dataBuffer.writeInt(4);
    dataBuffer.writeInt(0);
    dataBuffer.writeInt(12);
    dataBuffer.position = dataBuffer.position + 64; // /* reserved? 4 + 12 doublewords */
    // note, if the position wouldn't shift forward, write zeroes manually
    dataBuffer.writeShort(0xca01); // out_uint16_le(s, 0xca01);
    dataBuffer.writeShort(1);
    if (true) //Options.use_rdp5)
    {
        dataBuffer.writeInt(0); // out_uint32(s, 0);
        dataBuffer.writeByte(24); // out_uint8(s, g_server_bpp);
        dataBuffer.writeShort(0x0700); // out_uint16_le(s, 0x0700);
        dataBuffer.writeByte(0); // out_uint8(s, 0);
        dataBuffer.writeInt(1); // out_uint32_le(s, 1);
        dataBuffer.position = dataBuffer.position + 64;
        dataBuffer.writeShort(SEC_TAG_CLI_4); // out_uint16_le(s,
        // SEC_TAG_CLI_4);
        dataBuffer.writeShort(12); // out_uint16_le(s, 12);
        dataBuffer.writeInt(0xd); // out_uint32_le(s,
        // g_console_session
        // ?
        // 0xb
        // :
        // 9);
        // the comments say 9, but the code says 0xd - leaving 0xd in place
        // Options.console_session is hardcoded false
        dataBuffer.writeInt(0); // out_uint32(s, 0);
    }
    // Client encryption settings //
    dataBuffer.writeShort(SEC_TAG_CLI_CRYPT);
    dataBuffer.writeShort(12); // length
    // if(Options.use_rdp5) dataBuffer.setLittleEndian32(Options.encryption ?
    // 0x1b : 0); // 128-bit encryption supported
    // else
    dataBuffer.writeInt(true ? (false ? 0xb : 0x3) : 0);
    dataBuffer.writeInt(0); // unknown
    if (true && (num_channels > 0)) {
        trace(("num_channels is", num_channels));
        dataBuffer.writeShort(SEC_TAG_CLI_CHANNELS); // out_uint16_le(s,
        // SEC_TAG_CLI_CHANNELS);
        dataBuffer.writeShort(num_channels * 12 + 8); // out_uint16_le(s,
        // g_num_channels
        // * 12
        // + 8);
        // //
        // length
        dataBuffer.writeInt(num_channels); // out_uint32_le(s,
        // g_num_channels);
        // // number of
        // virtual
        // channels
        for (var i: int = 0; i < num_channels; i++) {
            targetPos=dataBuffer.position+8; // account for padding/truncation
            dataBuffer.writeMultiByte("testtes" + i, "ascii"); //, 8); // out_uint8a(s,
            // g_channels[i].name,
            // 8);
            dataBuffer.position=targetPos;
            dataBuffer.writeInt(0x00000040); // out_uint32_be(s,
            // g_channels[i].flags);
            // writing big-endian 0x40000000
        }
    }
    trace("sendMCSData: Data buffer length is",dataBuffer.length); // debug
    return dataBuffer;
}

Hope this helps.