diff --git a/source/_posts/tensorflow.md b/source/_posts/tensorflow.md index 3bda39b..ab38063 100644 --- a/source/_posts/tensorflow.md +++ b/source/_posts/tensorflow.md @@ -7,8 +7,7 @@ tags: - Python categories: - Other -intro: This is a quick reference list of cheat sheets for Tensorflow. See also [Tensorflow website](https://tensorflow.org/). This docs will -also include the usage of keras for the high-level models +intro: This is a quick reference list of cheat sheets for Tensorflow. See also [Tensorflow website](https://tensorflow.org/). --- ## Imports { .cols-1 } @@ -18,71 +17,29 @@ also include the usage of keras for the high-level models ``` import tensorflow as tf # root package import tensorflow_datasets as tfds # dataset representation and loading -tfds.load("mnist") # data here could be -``` - -### Neural Network API - -``` -import torch.autograd as autograd # computation graph -from torch import Tensor # tensor node in the computation graph -import torch.nn as nn # neural networks -import torch.nn.functional as F # layers, activations and more -import torch.optim as optim # optimizers e.g. gradient descent, ADAM, etc. -from torch.jit import script, trace # hybrid frontend decorator and tracing jit -``` - -### Torchscript and JIT - -``` -torch.jit.trace() # takes your module or function and an example - # data input, and traces the computational steps - # that the data encounters as it progresses through the model - -@script # decorator used to indicate data-dependent - # control flow within the code being traced -``` - -### ONNX - -``` -torch.onnx.export(model, dummy data, xxxx.proto) # exports an ONNX formatted - # model using a trained model, dummy - # data and the desired file name - -model = onnx.load("alexnet.proto") # load an ONNX model -onnx.checker.check_model(model) # check that the model - # IR is well formed - -onnx.helper.printable_graph(model.graph) # print a human readable - # representation of the graph -``` - -### Vision - -``` -from torchvision import datasets, models, transforms # vision datasets, - # architectures & - # transforms - -import torchvision.transforms as transforms # composable transforms -``` - -### Distributed Training - -``` -import torch.distributed as dist # distributed communication -from torch.multiprocessing import Process # memory sharing processes +model.compile(optimizer, loss, metrics) # compile necessary components for training and evaluation +model.fit(x_train, y_train, epoch, batch_size) # model training +model.evaluate(x_test, y_test) # model evaluation ``` ## Tensors { .cols-1 } +### Basic Operations +``` +a = tf.constant(5) + tf.constant(3) # tf.constant is an immutable tensor storing the fixed value +a.numpy() # This will return the value, which is 8 +b = tf.Variable(10) # tf.Variable is a shared state for an entire execution time +b.assign(15) # this assign the new value to the variable +with tf.GradientTape() as tape: # record operations on variables for automatic differentiation +``` + ### Creation ``` -x = torch.randn(*size) # tensor with independent N(0,1) entries -x = torch.[ones|zeros](*size) # tensor with all 1's [or 0's] -x = torch.tensor(L) # create tensor from [nested] list or ndarray L +x = +tf.random_normal_initializer(mean, std) # tensor with independent N(mean,stf) entries +tf.random_uniform_initializer(min_val, max_val) # tensor with independent Uniform(min_val, max_val) entries +x = tf.[ones|zeros](*size) # tensor with all 1's [or 0's] y = x.clone() # clone of x with torch.no_grad(): # code wrap that stops autograd from tracking tensor history requires_grad=True # arg, when set to True, tracks computation @@ -92,81 +49,79 @@ requires_grad=True # arg, when set to True, tracks computation ### Dimensionality ``` -x.size() # return tuple-like object of dimensions -x = torch.cat(tensor_seq, dim=0) # concatenates tensors along dim -y = x.view(a,b,...) # reshapes x into size (a,b,...) -y = x.view(-1,a) # reshapes x into size (b,a) for some b -y = x.transpose(a,b) # swaps dimensions a and b -y = x.permute(*dims) # permutes dimensions -y = x.unsqueeze(dim) # tensor with added axis -y = x.unsqueeze(dim=2) # (a,b,c) tensor -> (a,b,1,c) tensor -y = x.squeeze() # removes all dimensions of size 1 (a,1,b,1) -> (a,b) -y = x.squeeze(dim=1) # removes specified dimension of size 1 (a,1,b,1) -> (a,b,1) +tf.shape # shape of the tensor +tf.rank # number of dimension of the tensors +tf.size # number of elements in the tensor? +x = tf.concat(tensor_seq, axis=0) # concatenates tensors along axis +y = tf.reshape(tensor, [new_shape]) # reshapes x into size (a,b,...) +y = tf.reshape(tensor, [(-1,a]) # reshapes x into size (b,a) for some b +y = x.permute(*dims) # permutes dimensions +y = tf.expand_dims(x) # tensor with added axis +y = tf.expand_dims(x, axis=2) # (a,b,c) tensor -> (a,b,1,c) tensor ``` ### Algebra ``` -ret = A.mm(B) # matrix multiplication -ret = A.mv(x) # matrix-vector multiplication -x = x.t() # matrix transpose +tf.add(a, b), a + b # matrix addition +tf.multiply(a, b), a * b # matrix-vector multiplication +tf.matmul(a, b), a @ b # matrix multiplication +tf.transpose() # matrix transpose ``` ### GPU Usage ``` -torch.cuda.is_available # check for cuda -x = x.cuda() # move x's data from - # CPU to GPU and return new object +gpus = tf.config.list_physical_devices('GPU') # check whether there is a GPU usage +if gpus: -x = x.cpu() # move x's data from GPU to CPU - # and return new object +tf.device() # manual device placement + # either "/CPU:0", "/GPU:0", or other qualified name + # of the second GPU of your machine -if not args.disable_cuda and torch.cuda.is_available(): # device agnostic code - args.device = torch.device('cuda') # and modularity -else: # - args.device = torch.device('cpu') # +try: + tf.config.set_visible_devices(gpus[0], 'GPU') # Limiting GPU memory growth -net.to(device) # recursively convert their - # parameters and buffers to - # device specific tensors - -x = x.to(device) # copy your tensors to a device - # (gpu, cpu) -``` - -### Deep Learning ``` -nn.Linear(m,n) # fully connected layer from - # m to n units -nn.ConvXd(m,n,s) # X dimensional conv layer from - # m to n channels where X⍷{1,2,3} - # and the kernel size is s +## Deep Learning Models { .cols-1 } -nn.MaxPoolXd(s) # X dimension pooling layer - # (notation as above) +### Creating Models +``` +tf.keras.Sequential # stack layers in a way that the computation + # will be performed sequentially +``` -nn.BatchNormXd # batch norm layer -nn.RNN/LSTM/GRU # recurrent layers -nn.Dropout(p=0.5, inplace=False) # dropout layer for any dimensional input -nn.Dropout2d(p=0.5, inplace=False) # 2-dimensional channel-wise dropout -nn.Embedding(num_embeddings, embedding_dim) # (tensor-wise) mapping from - # indices to embedding vectors +### Layers + +``` +tf.keras.layers.Dense(m,n) # fully connected layer from + # m to n units + +tf.keras.layers.ConvXd(m,n,s) # X dimensional conv layer from + # m to n channels where X⍷{1,2,3} + # and the kernel size is s + +tf.keras.layers.MaxPoolXd(s) # X dimension pooling layer + # (notation as above) + +tf.keras.layers.BatchNormalization # batch norm layer +tf.keras.layers.RNN/LSTM/GRU # recurrent layers +tf.keras.layers.Dropout(rate=0.5) # dropout layer for any dimensional input +tf.keras.layers.Embedding(input_dim, output_dim) # (tensor-wise) mapping from + # indices to embedding vectors ``` ### Loss Functions ``` -nn.X # where X is L1Loss, MSELoss, CrossEntropyLoss - # CTCLoss, NLLLoss, PoissonNLLLoss, - # KLDivLoss, BCELoss, BCEWithLogitsLoss, - # MarginRankingLoss, HingeEmbeddingLoss, - # MultiLabelMarginLoss, SmoothL1Loss, - # SoftMarginLoss, MultiLabelSoftMarginLoss, - # CosineEmbeddingLoss, MultiMarginLoss, - # or TripletMarginLoss +tf.keras.losses.X # where X is BinaryCrossentropy, BinaryFocalCrossentropy, CTC + # CategoricalCrossentropy, CategoricalFocalCrossentropy, + # CategoricalHinge, CosineSimilarity, Dice, Hinge, Huber + # KLDivergence, LogCosh, MeanAbsoluteError, MeanAbsolutePercentageError + # MeanSquaredError, MeanSquaredLogarithmicError, Poisson + # Reduction, SparseCategoricalCrossentropy, SquaredHinge, Tversky ``` ### Activation Functions @@ -182,22 +137,27 @@ nn.X # where X is ReLU, ReLU6, ELU, SELU, PReLU ### Optimizers ``` -opt = optim.x(model.parameters(), ...) # create optimizer +opt = tf.keras.optimizer.x(model.parameters(), ...) # create optimizer opt.step() # update weights -optim.X # where X is SGD, Adadelta, Adagrad, Adam, - # AdamW, SparseAdam, Adamax, ASGD, - # LBFGS, RMSprop or Rprop +optim.X # where X is SGD, Adadelta, Adafactor, + # Adagrad, Adam, AdamW, Adamax, Ftrl, Lion, + # LossScaleOptimizer ,RMSprop or Rprop ``` -### Learning rate scheduling +### Learning rate scheduling - Callbacks ``` -scheduler = optim.X(optimizer,...) # create lr scheduler -scheduler.step() # update lr after optimizer updates weights -optim.lr_scheduler.X # where X is LambdaLR, MultiplicativeLR, - # StepLR, MultiStepLR, ExponentialLR, - # CosineAnnealingLR, ReduceLROnPlateau, CyclicLR, - # OneCycleLR, CosineAnnealingWarmRestarts, +callbacks = tf.keras.callbacks.LearningRateScheduler(scheduler) # create lr scheduler +model.fit(..., callbacks=[callback], ....) # update lr after optimizer updates weights + # using with fit(), evaluate(), and predict() +``` + +### Saving and Loading Models +``` +tf.keras.models.clone_model(...) # Clone a Functional or Sequential Model instance. +tf.keras.models.load_model(...) # Loads a model saved via model.save(). +tf.keras.models.model_from_json(...) # Parses a JSON model configuration string and returns a model instance. +tf.keras.models.save_model(...) # Saves a model as a .keras file. ``` ## Data Utilities { .cols-1 } @@ -205,20 +165,6 @@ optim.lr_scheduler.X # where X is LambdaLR, MultiplicativeLR, ### Datasets ``` -Dataset # abstract class representing dataset -TensorDataset # labelled dataset in the form of tensors -Concat Dataset # concatenation of Datasets -``` - -### Dataloaders and DataSamplers - -``` -DataLoader(dataset, batch_size=1, ...) # loads data batches agnostic - # of structure of individual data points - -sampler.Sampler(dataset,...) # abstract class dealing with - # ways to sample from dataset - -sampler.XSampler where ... # Sequential, Random, SubsetRandom, - # WeightedRandom, Batch, Distributed +pip install tensorflow-datasets # install the module +tfds.load('mnist', split, shuffle_files) # loading a dataset ```