CLD network protocol
From Hail Cloud Computing Wiki
OBSOLETE CONTENT
This wiki has been archived and the content is no longer updated.
CLD network protocol
Contents |
Overview
The network protocol used for communication between a CLD client and a CLD master server is a custom, message-based binary UDP protocol.
Axioms
sequence numbers
- 64-bit sequence numbers are used to create ordering in the stream. Each message is processed, and responded-to, in order.
- Client-to-server and server-to-client sequence numbers are independent from each other.
- Sequence numbers are initialized with a NEW-SESS message (client-to-server sequence number), and the response to NEW-SESS (server-to-client sequence number).
client
- Each packet from the client is sent to the server until acknowledged, or timeout.
- For each packet, client receives from server
- a message response, from a completed message operation, or
- an ACK-FRAG message, for all but the last fragment of a multi-packet message.
server
- Each packet from the server is sent to the client until acknowledged, or timeout.
- For each packet, server receives from client an ACK message.
Security
Each packet is individually cryptographically-signed using SHA-1 HMAC. Although the entire packet is signed, the primary components in the signature are
- a username,
- the sequence number (hopefully a nonce value; it is initialized from a random number), and
- a shared secret key:
/** header for each packet */
struct cld_packet {
...
uint64_t seqid; /**< sequence id (initialized by a random number) */
...
char user[CLD_MAX_USERNAME]; /**< authenticated user */
};
Protocol constants
/** available RPC operations */
enum cld_msg_ops {
/* client -> server */
cmo_nop = 0, /**< no op */
cmo_new_sess = 1, /**< new session */
cmo_open = 2, /**< open file */
cmo_get_meta = 3, /**< get metadata */
cmo_get = 4, /**< get metadata + data */
cmo_put = 6, /**< put data */
cmo_close = 7, /**< close file */
cmo_del = 8, /**< delete file */
cmo_lock = 9, /**< lock */
cmo_unlock = 10, /**< unlock */
cmo_trylock = 11, /**< trylock */
cmo_ack = 12, /**< ack of seqid rx'd */
cmo_end_sess = 13, /**< end session */
/* server -> client */
cmo_ping = 30, /**< server to client ping */
cmo_not_master = 31, /**< I am not the master! */
cmo_event = 32, /**< server->cli async event */
cmo_ack_frag = 33, /**< ack partial msg */
};
/** CLD error codes */
enum cle_err_codes {
CLE_OK = 0, /**< success / no error */
CLE_SESS_EXISTS = 1, /**< session exists */
CLE_SESS_INVAL = 2, /**< session doesn't exist */
CLE_DB_ERR = 3, /**< db error */
CLE_BAD_PKT = 4, /**< invalid/corrupted packet */
CLE_INODE_INVAL = 5, /**< inode doesn't exist */
CLE_NAME_INVAL = 6, /**< inode name invalid */
CLE_OOM = 7, /**< server out of memory */
CLE_FH_INVAL = 8, /**< file handle invalid */
CLE_DATA_INVAL = 9, /**< invalid data pkt */
CLE_LOCK_INVAL = 10, /**< invalid lock */
CLE_LOCK_CONFLICT = 11, /**< conflicting lock held */
CLE_LOCK_PENDING = 12, /**< lock waiting to be acq. */
CLE_MODE_INVAL = 13, /**< op incompat. w/ file mode */
CLE_INODE_EXISTS = 14, /**< inode exists */
CLE_DIR_NOTEMPTY = 15, /**< dir not empty */
CLE_INTERNAL_ERR = 16, /**< nonspecific internal err */
CLE_TIMEOUT = 17, /**< session timed out */
CLE_SIG_INVAL = 18, /**< HMAC sig bad / auth failed */
};
/** availble OPEN mode flags */
enum cld_open_modes {
COM_READ = (1 << 0), /**< read */
COM_WRITE = (1 << 1), /**< write */
COM_LOCK = (1 << 2), /**< lock */
COM_ACL = (1 << 3), /**< ACL update */
COM_CREATE = (1 << 4), /**< create file, if not exist */
COM_EXCL = (1 << 5), /**< fail create if file exists */
COM_DIRECTORY = (1 << 6), /**< operate on a directory */
};
/** potential events client may receive */
enum cld_events {
CE_UPDATED = (1 << 0), /**< contents updated */
CE_DELETED = (1 << 1), /**< inode deleted */
CE_LOCKED = (1 << 2), /**< lock acquired */
CE_MASTER_FAILOVER = (1 << 3), /**< master failover */
CE_SESS_FAILED = (1 << 4),
};
/** LOCK flags */
enum cld_lock_flags {
CLF_SHARED = (1 << 0), /**< a shared (read) lock */
};
/** CLD packet flags */
enum cld_packet_flags {
CPF_FIRST = (1 << 0), /**< first fragment */
CPF_LAST = (1 << 1), /**< last fragment */
};
Protocol structures
/** header for each packet */
struct cld_packet {
uint8_t magic[CLD_MAGIC_SZ]; /**< magic number; constant */
uint64_t seqid; /**< sequence id */
uint8_t sid[CLD_SID_SZ]; /**< client id */
uint32_t flags; /**< CPF_xxx flags */
uint8_t res[4];
char user[CLD_MAX_USERNAME]; /**< authenticated user */
};
/** header for each message */
struct cld_msg_hdr {
uint8_t magic[CLD_MAGIC_SZ]; /**< magic number; constant */
uint64_t xid; /**< opaque message id */
uint8_t op; /**< operation code */
uint8_t res1[7];
};
/** standard response for each message */
struct cld_msg_resp {
struct cld_msg_hdr hdr;
uint32_t code; /**< error code, CLE_xxx */
uint32_t rsv; /**< reserved */
uint64_t xid_in; /**< C->S xid */
};
/** ACK-FRAG message */
struct cld_msg_ack_frag {
struct cld_msg_hdr hdr;
uint64_t seqid; /**< sequence id to ack */
};
/** OPEN message */
struct cld_msg_open {
struct cld_msg_hdr hdr;
uint32_t mode; /**< open mode, COM_xxx */
uint32_t events; /**< events mask, CE_xxx */
uint16_t name_len; /**< length of file name */
uint8_t res[6];
/* inode name */
};
/** OPEN message response */
struct cld_msg_open_resp {
struct cld_msg_resp resp;
uint64_t fh; /**< handle opened */
};
/** GET message */
struct cld_msg_get {
struct cld_msg_hdr hdr;
uint64_t fh; /**< open file handle */
};
/** GET message response */
struct cld_msg_get_resp {
struct cld_msg_resp resp;
uint64_t inum; /**< unique inode number */
uint32_t ino_len; /**< inode name len */
uint32_t size; /**< data size */
uint64_t version; /**< inode version */
uint64_t time_create; /**< creation time */
uint64_t time_modify; /**< last modification time */
uint32_t flags; /**< inode flags; CIFL_xxx */
uint8_t res[4];
/* inode name */
};
/** PUT message */
struct cld_msg_put {
struct cld_msg_hdr hdr;
uint64_t fh; /**< open file handle */
uint32_t data_size; /**< total size of data */
uint8_t res[4];
};
/** CLOSE message */
struct cld_msg_close {
struct cld_msg_hdr hdr;
uint64_t fh; /**< open file handle */
};
/** DEL message */
struct cld_msg_del {
struct cld_msg_hdr hdr;
uint16_t name_len; /**< length of file name */
uint8_t res[6];
/* inode name */
};
/** UNLOCK message */
struct cld_msg_unlock {
struct cld_msg_hdr hdr;
uint64_t fh; /**< open file handle */
};
/** LOCK message */
struct cld_msg_lock {
struct cld_msg_hdr hdr;
uint64_t fh; /**< open file handle */
uint32_t flags; /**< CLF_xxx */
uint8_t res[4];
};
/** Server-to-client EVENT message */
struct cld_msg_event {
struct cld_msg_hdr hdr;
uint64_t fh; /**< open file handle */
uint32_t events; /**< CE_xxx */
uint8_t res[4];
};
