CLD network protocol

From Hail Cloud Computing Wiki
Jump to: navigation, search

CLD network protocol

Contents

Overview

The network protocol used for communication between a CLD client and a CLD master server is a custom, message-based binary UDP protocol.

Axioms

sequence numbers

  • 64-bit sequence numbers are used to create ordering in the stream. Each message is processed, and responded-to, in order.
  • Client-to-server and server-to-client sequence numbers are independent from each other.
  • Sequence numbers are initialized with a NEW-SESS message (client-to-server sequence number), and the response to NEW-SESS (server-to-client sequence number).

client

  • Each packet from the client is sent to the server until acknowledged, or timeout.
  • For each packet, client receives from server
    • a message response, from a completed message operation, or
    • an ACK-FRAG message, for all but the last fragment of a multi-packet message.

server

  • Each packet from the server is sent to the client until acknowledged, or timeout.
  • For each packet, server receives from client an ACK message.

Security

Each packet is individually cryptographically-signed using SHA-1 HMAC. Although the entire packet is signed, the primary components in the signature are

  • a username,
  • the sequence number (hopefully a nonce value; it is initialized from a random number), and
  • a shared secret key:
/** header for each packet */
struct cld_packet {
...
        uint64_t        seqid;                  /**< sequence id (initialized by a random number) */
...
        char            user[CLD_MAX_USERNAME]; /**< authenticated user */
};  


Protocol constants

/** available RPC operations */
enum cld_msg_ops {
        /* client -> server */
        cmo_nop                 = 0,            /**< no op */
        cmo_new_sess            = 1,            /**< new session */
        cmo_open                = 2,            /**< open file */
        cmo_get_meta            = 3,            /**< get metadata */
        cmo_get                 = 4,            /**< get metadata + data */
        cmo_put                 = 6,            /**< put data */
        cmo_close               = 7,            /**< close file */
        cmo_del                 = 8,            /**< delete file */
        cmo_lock                = 9,            /**< lock */
        cmo_unlock              = 10,           /**< unlock */
        cmo_trylock             = 11,           /**< trylock */
        cmo_ack                 = 12,           /**< ack of seqid rx'd */
        cmo_end_sess            = 13,           /**< end session */

        /* server -> client */
        cmo_ping                = 30,           /**< server to client ping */
        cmo_not_master          = 31,           /**< I am not the master! */
        cmo_event               = 32,           /**< server->cli async event */
        cmo_ack_frag            = 33,           /**< ack partial msg */
};

/** CLD error codes */
enum cle_err_codes {
        CLE_OK                  = 0,            /**< success / no error */
        CLE_SESS_EXISTS         = 1,            /**< session exists */
        CLE_SESS_INVAL          = 2,            /**< session doesn't exist */
        CLE_DB_ERR              = 3,            /**< db error */
        CLE_BAD_PKT             = 4,            /**< invalid/corrupted packet */
        CLE_INODE_INVAL         = 5,            /**< inode doesn't exist */
        CLE_NAME_INVAL          = 6,            /**< inode name invalid */
        CLE_OOM                 = 7,            /**< server out of memory */
        CLE_FH_INVAL            = 8,            /**< file handle invalid */
        CLE_DATA_INVAL          = 9,            /**< invalid data pkt */
        CLE_LOCK_INVAL          = 10,           /**< invalid lock */
        CLE_LOCK_CONFLICT       = 11,           /**< conflicting lock held */
        CLE_LOCK_PENDING        = 12,           /**< lock waiting to be acq. */
        CLE_MODE_INVAL          = 13,           /**< op incompat. w/ file mode */
        CLE_INODE_EXISTS        = 14,           /**< inode exists */
        CLE_DIR_NOTEMPTY        = 15,           /**< dir not empty */
        CLE_INTERNAL_ERR        = 16,           /**< nonspecific internal err */
        CLE_TIMEOUT             = 17,           /**< session timed out */
        CLE_SIG_INVAL           = 18,           /**< HMAC sig bad / auth failed  */
};

/** availble OPEN mode flags */
enum cld_open_modes {
        COM_READ                = (1 << 0),     /**< read */
        COM_WRITE               = (1 << 1),     /**< write */
        COM_LOCK                = (1 << 2),     /**< lock */
        COM_ACL                 = (1 << 3),     /**< ACL update */
        COM_CREATE              = (1 << 4),     /**< create file, if not exist */
        COM_EXCL                = (1 << 5),     /**< fail create if file exists */
        COM_DIRECTORY           = (1 << 6),     /**< operate on a directory */
};

/** potential events client may receive */
enum cld_events {
        CE_UPDATED              = (1 << 0),     /**< contents updated */
        CE_DELETED              = (1 << 1),     /**< inode deleted */
        CE_LOCKED               = (1 << 2),     /**< lock acquired */
        CE_MASTER_FAILOVER      = (1 << 3),     /**< master failover */
        CE_SESS_FAILED          = (1 << 4),
};

/** LOCK flags */
enum cld_lock_flags {
        CLF_SHARED              = (1 << 0),     /**< a shared (read) lock */
};

/** CLD packet flags */
enum cld_packet_flags {
        CPF_FIRST               = (1 << 0),     /**< first fragment */
        CPF_LAST                = (1 << 1),     /**< last fragment */
};


Protocol structures

/** header for each packet */
struct cld_packet {
       uint8_t         magic[CLD_MAGIC_SZ];    /**< magic number; constant */
       uint64_t        seqid;                  /**< sequence id */
       uint8_t         sid[CLD_SID_SZ];        /**< client id */
       uint32_t        flags;                  /**< CPF_xxx flags */
       uint8_t         res[4];
       char            user[CLD_MAX_USERNAME]; /**< authenticated user */
};

/** header for each message */
struct cld_msg_hdr {
       uint8_t         magic[CLD_MAGIC_SZ];    /**< magic number; constant */
       uint64_t        xid;                    /**< opaque message id */
       uint8_t         op;                     /**< operation code */
       uint8_t         res1[7];
};

/** standard response for each message */
struct cld_msg_resp {
       struct cld_msg_hdr      hdr;

       uint32_t                code;           /**< error code, CLE_xxx */
       uint32_t                rsv;            /**< reserved */
       uint64_t                xid_in;  /**< C->S xid */
};

/** ACK-FRAG message */
struct cld_msg_ack_frag {
       struct cld_msg_hdr      hdr;

       uint64_t                seqid;          /**< sequence id to ack */
};

/** OPEN message */
struct cld_msg_open {
       struct cld_msg_hdr      hdr;

       uint32_t                mode;           /**< open mode, COM_xxx */
       uint32_t                events;  /**< events mask, CE_xxx */
       uint16_t                name_len;       /**< length of file name */
       uint8_t                 res[6];
       /* inode name */
};

/** OPEN message response */
struct cld_msg_open_resp {
       struct cld_msg_resp     resp;

       uint64_t                fh;             /**< handle opened */
};

/** GET message */
struct cld_msg_get {
       struct cld_msg_hdr      hdr;

       uint64_t                fh;             /**< open file handle */
};

/** GET message response */
struct cld_msg_get_resp {
       struct cld_msg_resp     resp;

       uint64_t                inum;           /**< unique inode number */
       uint32_t                ino_len;        /**< inode name len */
       uint32_t                size;           /**< data size */
       uint64_t                version;        /**< inode version */
       uint64_t                time_create;    /**< creation time */
       uint64_t                time_modify;    /**< last modification time */
       uint32_t                flags;          /**< inode flags; CIFL_xxx */
       uint8_t                 res[4];
       /* inode name */
};

/** PUT message */
struct cld_msg_put {
       struct cld_msg_hdr      hdr;

       uint64_t                fh;             /**< open file handle */
       uint32_t                data_size;      /**< total size of data */
       uint8_t                 res[4];
};

/** CLOSE message */
struct cld_msg_close {
       struct cld_msg_hdr      hdr;

       uint64_t                fh;             /**< open file handle */
};

/** DEL message */
struct cld_msg_del {
       struct cld_msg_hdr      hdr;

       uint16_t                name_len;       /**< length of file name */
       uint8_t                 res[6];
       /* inode name */
};

/** UNLOCK message */
struct cld_msg_unlock {
       struct cld_msg_hdr      hdr;

       uint64_t                fh;             /**< open file handle */
};

/** LOCK message */
struct cld_msg_lock {
       struct cld_msg_hdr      hdr;

       uint64_t                fh;             /**< open file handle */
       uint32_t                flags;          /**< CLF_xxx */
       uint8_t                 res[4];
};

/** Server-to-client EVENT message */
struct cld_msg_event {
       struct cld_msg_hdr      hdr;

       uint64_t                fh;             /**< open file handle */
       uint32_t                events;  /**< CE_xxx */
       uint8_t                 res[4];
};
Personal tools