From e83a60167d5bced6fe9d69dd1348011aadf794c9 Mon Sep 17 00:00:00 2001 From: Josh Dersch Date: Thu, 4 Feb 2016 18:37:38 -0800 Subject: [PATCH] Implementation of FTP started. --- PUP/BSPManager.cs | 228 +++++++++++++++++++++++++-------- PUP/CopyDisk/CopyDiskServer.cs | 1 + PUP/Entrypoint.cs | 30 +---- PUP/FTP/FTPServer.cs | 193 ++++++++++++++++++++++++++++ PUP/IFS.csproj | 1 + PUP/Logging/Log.cs | 3 +- PUP/PUP.cs | 5 +- PUP/Serializer.cs | 33 ++++- 8 files changed, 409 insertions(+), 85 deletions(-) create mode 100644 PUP/FTP/FTPServer.cs diff --git a/PUP/BSPManager.cs b/PUP/BSPManager.cs index ad6533f..3761a0e 100644 --- a/PUP/BSPManager.cs +++ b/PUP/BSPManager.cs @@ -36,7 +36,7 @@ namespace IFS _outputLock = new ReaderWriterLockSlim(); _inputWriteEvent = new AutoResetEvent(false); - _inputQueue = new Queue(65536); + _inputQueue = new Queue(65536); _outputAckEvent = new AutoResetEvent(false); _outputQueue = new Queue(65536); @@ -71,6 +71,14 @@ namespace IFS get { return _serverConnectionPort; } } + /// + /// Returns the last Mark byte received, if any. + /// + public byte LastMark + { + get { return _lastMark; } + } + public void Destroy() { if (OnDestroy != null) @@ -103,43 +111,68 @@ namespace IFS /// /// Reads data from the channel (i.e. from the client). Will block if not all the requested data is available. + /// If a Mark byte is encountered, will return a short read. /// /// public int Read(ref byte[] data, int count, int offset) { // sanity check - if (count > data.Length) + if (count + offset > data.Length) { - throw new InvalidOperationException("count must be less than or equal to the length of the buffer being read into."); + throw new InvalidOperationException("count + offset must be less than or equal to the length of the buffer being read into."); } int read = 0; - // Loop until the data we asked for arrives or until we time out waiting. - // TODO: handle partial transfers due to aborted BSPs. - while (true) + // + // Loop until either: + // - all the data we asked for arrives + // - we get a Mark byte + // - we time out waiting for data + // + bool done = false; + while (!done) { _inputLock.EnterUpgradeableReadLock(); - if (_inputQueue.Count >= count) + if (_inputQueue.Count > 0) { _inputLock.EnterWriteLock(); - // We have the data right now, read it and return. - // TODO: this is a really inefficient thing. - for (int i = 0; i < count; i++) - { - data[i + offset] = _inputQueue.Dequeue(); - } - - _inputLock.ExitWriteLock(); - _inputLock.ExitUpgradeableReadLock(); - break; + // We have some data right now, read it in. + // TODO: this code is ugly and it wants to die. + while (_inputQueue.Count > 0 && read < count) + { + ushort word = _inputQueue.Dequeue(); + + // Is this a mark or a data byte? + if (word < 0x100) + { + // Data, place in data stream + data[read + offset] = (byte)word; + read++; + } + else + { + // Mark. Set last mark and exit. + _lastMark = (byte)(word >> 8); + done = true; + break; + } + } + + if (read >= count) + { + done = true; + } + + _inputLock.ExitWriteLock(); + _inputLock.ExitUpgradeableReadLock(); } else { _inputLock.ExitUpgradeableReadLock(); - // Not enough data in the queue. + // No data in the queue. // Wait until we have received more data, then try again. if (!_inputWriteEvent.WaitOne(BSPReadTimeoutPeriod)) { @@ -180,10 +213,53 @@ namespace IFS } /// - /// Appends incoming client data into the input queue (called from BSPManager to place new PUP data into the BSP stream) + /// Reads data from the queue until a Mark byte is received. + /// The mark byte is returned (LastMark is also set.) + /// + /// + public byte WaitForMark() + { + byte mark = 0; + + // This data is discarded. The length is arbitrary. + byte[] dummyData = new byte[512]; + + while(true) + { + int read = Read(ref dummyData, dummyData.Length); + + // Short read, indicating a Mark. + if (read < dummyData.Length) + { + mark = _lastMark; + break; + } + } + + return mark; + } + + /// + /// Appends incoming client data or Marks into the input queue (called from BSPManager to place new PUP data into the BSP stream) /// public void RecvWriteQueue(PUP dataPUP) { + // + // Sanity check: If this is a Mark PUP, the contents must only be one byte in length. + // + bool markPup = dataPUP.Type == PupType.AMark || dataPUP.Type == PupType.Mark; + if (markPup) + { + if (dataPUP.Contents.Length != 1) + { + Log.Write(LogType.Error, LogComponent.BSP, "Mark PUP must be 1 byte in length."); + + SendAbort("Mark PUP must be 1 byte in length."); + BSPManager.DestroyChannel(this); + return; + } + } + // If we are over our high watermark, we will drop the data (and not send an ACK even if requested). // Clients should be honoring the limits we set in the RFC packets. _inputLock.EnterUpgradeableReadLock(); @@ -211,15 +287,24 @@ namespace IFS // Prepare to add data to the queue - // Again, this is really inefficient + _inputLock.EnterWriteLock(); - for (int i = 0; i < dataPUP.Contents.Length; i++) + if (markPup) { - _inputQueue.Enqueue(dataPUP.Contents[i]); - - //Console.Write("{0:x} ({1}), ", dataPUP.Contents[i], (char)dataPUP.Contents[i]); - } + // + // For mark pups, the data goes in the high byte of the word + // so that it can be identified as a mark when it's read back. + _inputQueue.Enqueue((ushort)(dataPUP.Contents[0] << 8)); + } + else + { + // Again, this is really inefficient + for (int i = 0; i < dataPUP.Contents.Length; i++) + { + _inputQueue.Enqueue(dataPUP.Contents[i]); + } + } _recv_pos += (UInt32)dataPUP.Contents.Length; @@ -230,11 +315,10 @@ namespace IFS _inputWriteEvent.Set(); // If the client wants an ACK, send it now. - if ((PupType)dataPUP.Type == PupType.AData) + if (dataPUP.Type == PupType.AData || dataPUP.Type == PupType.AMark) { SendAck(); } - } /// @@ -274,31 +358,9 @@ namespace IFS } // Send the data, retrying as necessary. - int retry; - for (retry = 0; retry < BSPRetryCount; retry++) - { - PUP dataPup = new PUP(PupType.AData, _send_pos, _clientConnectionPort, _serverConnectionPort, chunk); - PUPProtocolDispatcher.Instance.SendPup(dataPup); - - _send_pos += (uint)chunk.Length; - - // Await an ack for the PUP we just sent. If we timeout, we will retry. - // - if (_outputAckEvent.WaitOne(BSPAckTimeoutPeriod)) - { - break; - } - - Log.Write(LogType.Warning, LogComponent.BSP, "ACK not received for sent data, retrying."); - } - - if (retry >= BSPRetryCount) - { - Log.Write(LogType.Error, LogComponent.BSP, "ACK not received after retries, aborting connection."); - SendAbort("ACK not received for sent data."); - BSPManager.DestroyChannel(this); - } - + PUP dataPup = new PUP(PupType.AData, _send_pos, _clientConnectionPort, _serverConnectionPort, chunk); + _send_pos += (uint)chunk.Length; + SendPupAwaitAck(dataPup); } } } @@ -309,6 +371,23 @@ namespace IFS PUPProtocolDispatcher.Instance.SendPup(abortPup); } + public void SendMark(byte markType, bool ack) + { + PUP markPup = new PUP(ack ? PupType.AMark : PupType.Mark, _send_pos, _clientConnectionPort, _serverConnectionPort, new byte[] { markType }); + + // Move pointer one byte for the Mark. + _send_pos++; + + if (ack) + { + SendPupAwaitAck(markPup); + } + else + { + PUPProtocolDispatcher.Instance.SendPup(markPup); + } + } + /// /// Invoked when the client sends an ACK /// @@ -368,6 +447,37 @@ namespace IFS PUPProtocolDispatcher.Instance.SendPup(ackPup); } + /// + /// Sends a PUP and waits for acknowledgement. On timeout, will retry. + /// If all retries fails, the channel is closed. + /// + /// + private void SendPupAwaitAck(PUP p) + { + // Send the data, retrying as necessary. + int retry; + for (retry = 0; retry < BSPRetryCount; retry++) + { + PUPProtocolDispatcher.Instance.SendPup(p); + + // Await an ack for the PUP we just sent. If we timeout, we will retry. + // + if (_outputAckEvent.WaitOne(BSPAckTimeoutPeriod)) + { + break; + } + + Log.Write(LogType.Warning, LogComponent.BSP, "ACK not received for sent data, retrying."); + } + + if (retry >= BSPRetryCount) + { + Log.Write(LogType.Error, LogComponent.BSP, "ACK not received after retries, aborting connection."); + SendAbort("ACK not received for sent data."); + BSPManager.DestroyChannel(this); + } + } + private BSPProtocol _protocolHandler; private UInt32 _recv_pos; @@ -384,10 +494,13 @@ namespace IFS private System.Threading.AutoResetEvent _outputAckEvent; - // TODO: replace this with a more efficient structure for buffering data - private Queue _inputQueue; + // NOTE: The input queue consists of ushorts so that + // we can encapsulate Mark bytes without using a separate data structure. + private Queue _inputQueue; private Queue _outputQueue; + private byte _lastMark; + // Constants // For now, we work on one PUP at a time. @@ -501,6 +614,13 @@ namespace IFS DestroyChannel(channel); } break; + + case PupType.Mark: + case PupType.AMark: + { + channel.RecvWriteQueue(p); + } + break; case PupType.Abort: { diff --git a/PUP/CopyDisk/CopyDiskServer.cs b/PUP/CopyDisk/CopyDiskServer.cs index a3246aa..7e74bb3 100644 --- a/PUP/CopyDisk/CopyDiskServer.cs +++ b/PUP/CopyDisk/CopyDiskServer.cs @@ -205,6 +205,7 @@ namespace IFS.CopyDisk public override void InitializeServerForChannel(BSPChannel channel) { // Spawn new worker + // TODO: keep track of workers to allow clean shutdown, management, etc. CopyDiskWorker worker = new CopyDiskWorker(channel); } } diff --git a/PUP/Entrypoint.cs b/PUP/Entrypoint.cs index 7499e46..fab821e 100644 --- a/PUP/Entrypoint.cs +++ b/PUP/Entrypoint.cs @@ -1,4 +1,5 @@ using IFS.CopyDisk; +using IFS.FTP; using IFS.Transport; using System; using System.Collections.Generic; @@ -9,33 +10,9 @@ using System.Threading.Tasks; namespace IFS { public class Entrypoint - { - struct foo - { - public ushort Bar; - public short Baz; - public byte Thing; - public int Inty; - public uint Uinty; - public BCPLString Quux; - } - + { static void Main(string[] args) - { - foo newFoo = new foo(); - newFoo.Bar = 0x1234; - newFoo.Baz = 0x5678; - newFoo.Thing = 0xcc; - newFoo.Inty = 0x01020304; - newFoo.Uinty = 0x05060708; - newFoo.Quux = new BCPLString("The quick brown fox jumped over the lazy dog's tail."); - - byte[] data = Serializer.Serialize(newFoo); - - - foo oldFoo = (foo) Serializer.Deserialize(data, typeof(foo)); - - + { List ifaces = EthernetInterface.EnumerateDevices(); Console.WriteLine("available interfaces are:"); @@ -53,6 +30,7 @@ namespace IFS // RTP/BSP based: PUPProtocolDispatcher.Instance.RegisterProtocol(new PUPProtocolEntry("CopyDisk", 0x15 /* 25B */, ConnectionType.BSP, new CopyDiskServer())); + PUPProtocolDispatcher.Instance.RegisterProtocol(new PUPProtocolEntry("FTP", 0x3, ConnectionType.BSP, new FTPServer())); // TODO: MAKE THIS CONFIGURABLE. PUPProtocolDispatcher.Instance.RegisterInterface(ifaces[2]); diff --git a/PUP/FTP/FTPServer.cs b/PUP/FTP/FTPServer.cs new file mode 100644 index 0000000..a667129 --- /dev/null +++ b/PUP/FTP/FTPServer.cs @@ -0,0 +1,193 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; + +using IFS.Logging; + +namespace IFS.FTP +{ + public enum FTPCommand + { + Invalid = 0, + Retrieve = 1, + Store = 2, + NewStore = 9, + Yes = 3, + No = 4, + HereIsPropertyList = 11, + HereIsFile = 5, + Version = 8, + Comment = 7, + EndOfCommand = 6, + Enumerate = 10, + NewEnumerate = 12, + Delete = 14, + Rename = 15, + } + + struct FTPVersion + { + public FTPVersion(byte version, string herald) + { + Version = version; + Herald = herald; + } + + public byte Version; + public string Herald; + } + + public class FTPServer : BSPProtocol + { + /// + /// Called by dispatcher to send incoming data destined for this protocol. + /// + /// + public override void RecvData(PUP p) + { + throw new NotImplementedException(); + } + + public override void InitializeServerForChannel(BSPChannel channel) + { + // Spawn new worker + FTPWorker ftpWorker = new FTPWorker(channel); + } + } + + public class FTPWorker + { + public FTPWorker(BSPChannel channel) + { + // Register for channel events + channel.OnDestroy += OnChannelDestroyed; + + _running = true; + + _workerThread = new Thread(new ParameterizedThreadStart(FTPWorkerThreadInit)); + _workerThread.Start(channel); + } + + private void OnChannelDestroyed() + { + // Tell the thread to exit and give it a short period to do so... + _running = false; + + Log.Write(LogType.Verbose, LogComponent.FTP, "Asking FTP worker thread to exit..."); + _workerThread.Join(1000); + + if (_workerThread.IsAlive) + { + Logging.Log.Write(LogType.Verbose, LogComponent.FTP, "FTP worker thread did not exit, terminating."); + _workerThread.Abort(); + } + } + + private void FTPWorkerThreadInit(object obj) + { + _channel = (BSPChannel)obj; + // + // Run the worker thread. + // If anything goes wrong, log the exception and tear down the BSP connection. + // + try + { + FTPWorkerThread(); + } + catch (Exception e) + { + if (!(e is ThreadAbortException)) + { + Log.Write(LogType.Error, LogComponent.FTP, "FTP worker thread terminated with exception '{0}'.", e.Message); + _channel.SendAbort("Server encountered an error."); + } + } + } + + private void FTPWorkerThread() + { + // Buffer used to receive command data. + byte[] buffer = new byte[1024]; + + while (_running) + { + // Discard input until we get a Mark. We should (in general) get a + // command, followed by EndOfCommand. + FTPCommand command = (FTPCommand)_channel.WaitForMark(); + + // Read data until the next Mark, which should be "EndOfCommand" + int length = _channel.Read(ref buffer, buffer.Length); + + // + // Sanity check: FTP spec doesn't specify max length of a command so the current + // length is merely a guess. If we actually filled the buffer then we should note it + // so this can be corrected. + // + if (length == buffer.Length) + { + throw new InvalidOperationException("Expected short read for FTP command."); + } + + // + // Ensure that next Mark is "EndOfCommand" + // + if (_channel.LastMark != (byte)FTPCommand.EndOfCommand) + { + throw new InvalidOperationException(String.Format("Expected EndOfCommand, got {0}", (FTPCommand)_channel.LastMark)); + } + + // + // TODO: this is ugly, figure out a clean way to do this. We need to deal with only the + // actual data retrieved. Due to the clumsy way we read it in we need to copy it here. + // + byte[] data = new byte[length]; + Array.Copy(buffer, data, length); + + // + // At this point we should have the entire command, execute it. + // + switch(command) + { + case FTPCommand.Version: + { + FTPVersion version = (FTPVersion)Serializer.Deserialize(data, typeof(FTPVersion)); + Log.Write(LogType.Normal, LogComponent.FTP, "Client FTP version is {0}, herald is '{1}.", version.Version, version.Herald); + + // + // Return our Version. + FTPVersion serverVersion = new FTPVersion(1, "LCM IFS FTP of 4 Feb 2016."); + SendFTPResponse(FTPCommand.Version, serverVersion); + } + break; + + case FTPCommand.Enumerate: + { + // Argument to Enumerate is a property list (string). + // + string fileSpec = Helpers.ArrayToString(data); + Log.Write(LogType.Verbose, LogComponent.FTP, "File spec for enumeration is '{0}.", fileSpec); + } + break; + + default: + Log.Write(LogType.Warning, LogComponent.FTP, "Unhandled FTP command {0}.", command); + break; + } + } + } + + private void SendFTPResponse(FTPCommand responseCommand, object data) + { + _channel.SendMark((byte)FTPCommand.Version, false); + _channel.Send(Serializer.Serialize(data)); + _channel.SendMark((byte)FTPCommand.EndOfCommand, true); + } + + private BSPChannel _channel; + + private Thread _workerThread; + private bool _running; + } +} diff --git a/PUP/IFS.csproj b/PUP/IFS.csproj index f9fb74f..80d3625 100644 --- a/PUP/IFS.csproj +++ b/PUP/IFS.csproj @@ -80,6 +80,7 @@ + diff --git a/PUP/Logging/Log.cs b/PUP/Logging/Log.cs index 2986272..c92c46d 100644 --- a/PUP/Logging/Log.cs +++ b/PUP/Logging/Log.cs @@ -19,6 +19,7 @@ namespace IFS.Logging CopyDisk = 0x10, DirectoryServices = 0x20, PUP = 0x40, + FTP = 0x80, All = 0x7fffffff } @@ -46,7 +47,7 @@ namespace IFS.Logging { // TODO: make configurable _components = LogComponent.All; - _type = LogType.Normal; + _type = LogType.All; //_logStream = new StreamWriter("log.txt"); } diff --git a/PUP/PUP.cs b/PUP/PUP.cs index 8925465..e195934 100644 --- a/PUP/PUP.cs +++ b/PUP/PUP.cs @@ -189,10 +189,11 @@ namespace IFS Contents = new byte[contentLength]; contents.CopyTo(Contents, 0); - Length = (ushort)(PUP_HEADER_SIZE + PUP_CHECKSUM_SIZE + contentLength); + // Length is always the real length of the data (not padded to an even number) + Length = (ushort)(PUP_HEADER_SIZE + PUP_CHECKSUM_SIZE + contents.Length); // Stuff data into raw array - _rawData = new byte[Length]; + _rawData = new byte[PUP_HEADER_SIZE + PUP_CHECKSUM_SIZE + contentLength]; // // Subtract off one byte from the Length value if the contents contain a garbage byte. diff --git a/PUP/Serializer.cs b/PUP/Serializer.cs index 346782c..1c36611 100644 --- a/PUP/Serializer.cs +++ b/PUP/Serializer.cs @@ -55,6 +55,7 @@ namespace IFS // - int // - uint // - BCPLString + // - string (MUST be last field in struct, if present) // - byte[] // // Struct fields are serialized in the order they are defined in the struct. Only Public instance fields are considered. @@ -131,6 +132,25 @@ namespace IFS } break; + case "String": + { + // The field MUST be the last in the struct. + if (i != info.Length - 1) + { + throw new InvalidOperationException("Non-BCPL strings must be the last field in the struct to be deserialized."); + } + + StringBuilder sb = new StringBuilder((int)(ms.Length - ms.Position)); + + while (ms.Position != ms.Length) + { + sb.Append((char)ms.ReadByte()); + } + + info[i].SetValue(o, sb.ToString()); + } + break; + default: throw new InvalidOperationException(String.Format("Type {0} is unsupported for deserialization.", info[i].FieldType.Name)); } @@ -147,7 +167,7 @@ namespace IFS public static byte[] Serialize(object o) { MemoryStream ms = new MemoryStream(); - + // // We support serialization of structs containing only: // - byte @@ -155,6 +175,8 @@ namespace IFS // - short // - int // - uint + // - string + // - byte[] // - BCPLString // // Struct fields are serialized in the order they are defined in the struct. Only Public instance fields are considered. @@ -232,7 +254,14 @@ namespace IFS ms.Write(value, 0, value.Length); } break; - + + case "String": + { + string value = (string)(info[i].GetValue(o)); + byte[] stringArray = Helpers.StringToArray(value); + ms.Write(stringArray, 0, stringArray.Length); + } + break; default: throw new InvalidOperationException(String.Format("Type {0} is unsupported for serialization.", info[i].FieldType.Name));