{
  "trx_id": "c12ec85524d57ce045b2b53e2d3b2ca961879844",
  "block": 106141280,
  "trx_in_block": 0,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2026-05-17T22:26:18",
  "op": [
    "delegate_vesting_shares",
    {
      "delegator": "steem",
      "delegatee": "calclavia",
      "vesting_shares": "7120.542580 VESTS"
    }
  ]
}

{
  "trx_id": "2ae526e97f5aab89f5ab090883b2a81c2c26d4c4",
  "block": 105967061,
  "trx_in_block": 1,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2026-05-11T20:36:45",
  "op": [
    "delegate_vesting_shares",
    {
      "delegator": "steem",
      "delegatee": "calclavia",
      "vesting_shares": "4408.332175 VESTS"
    }
  ]
}

{
  "trx_id": "bfe71fbd5f03781dcec33fb31b4580761626c775",
  "block": 105508984,
  "trx_in_block": 1,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2026-04-25T21:50:03",
  "op": [
    "delegate_vesting_shares",
    {
      "delegator": "steem",
      "delegatee": "calclavia",
      "vesting_shares": "7133.058336 VESTS"
    }
  ]
}

{
  "trx_id": "aed9118bc6ef54bdbf9c09536d46d8b9ff2bb2c4",
  "block": 102846089,
  "trx_in_block": 0,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2026-01-23T03:01:15",
  "op": [
    "delegate_vesting_shares",
    {
      "delegator": "steem",
      "delegatee": "calclavia",
      "vesting_shares": "4449.878994 VESTS"
    }
  ]
}

{
  "trx_id": "cde2d6bb195d33b6bf2cd2128257e0c34e038341",
  "block": 91292496,
  "trx_in_block": 4,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2024-12-16T22:20:36",
  "op": [
    "delegate_vesting_shares",
    {
      "delegator": "steem",
      "delegatee": "calclavia",
      "vesting_shares": "4614.098191 VESTS"
    }
  ]
}

{
  "trx_id": "072f0719c62d1bf8abd308ddcbca538b95c55940",
  "block": 79846757,
  "trx_in_block": 1,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2023-11-13T14:05:36",
  "op": [
    "delegate_vesting_shares",
    {
      "delegator": "steem",
      "delegatee": "calclavia",
      "vesting_shares": "4783.231723 VESTS"
    }
  ]
}

{
  "trx_id": "8c9a5a09da1bd8a440fc027dc520771f5116cef2",
  "block": 78345313,
  "trx_in_block": 6,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2023-09-21T19:43:21",
  "op": [
    "delegate_vesting_shares",
    {
      "delegator": "steem",
      "delegatee": "calclavia",
      "vesting_shares": "7720.510509 VESTS"
    }
  ]
}

{
  "trx_id": "0b475fce1d5f86534a6aa4c64d431ccf02777444",
  "block": 69110932,
  "trx_in_block": 7,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2022-11-03T09:45:09",
  "op": [
    "delegate_vesting_shares",
    {
      "delegator": "steem",
      "delegatee": "calclavia",
      "vesting_shares": "7942.191947 VESTS"
    }
  ]
}

{
  "trx_id": "3a77b7559471556f8b164b0c3b0ee44f300d9b85",
  "block": 60807293,
  "trx_in_block": 3,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2022-01-17T09:10:39",
  "op": [
    "delegate_vesting_shares",
    {
      "delegator": "steem",
      "delegatee": "calclavia",
      "vesting_shares": "8162.725178 VESTS"
    }
  ]
}

{
  "trx_id": "104a8f9c7314609e1d3f7f9d07dda7aa12e6d397",
  "block": 54605769,
  "trx_in_block": 7,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2021-06-13T23:10:12",
  "op": [
    "delegate_vesting_shares",
    {
      "delegator": "steem",
      "delegatee": "calclavia",
      "vesting_shares": "8346.493836 VESTS"
    }
  ]
}

{
  "trx_id": "1469c3393558c7c1e13519ab118d29e7864dd9da",
  "block": 49353292,
  "trx_in_block": 1,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2020-12-11T09:31:24",
  "op": [
    "delegate_vesting_shares",
    {
      "delegator": "steem",
      "delegatee": "calclavia",
      "vesting_shares": "8533.915810 VESTS"
    }
  ]
}

{
  "trx_id": "2d2ba7bc5992321738df562fa3dd5a1a49d2e398",
  "block": 49204861,
  "trx_in_block": 1,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2020-12-06T03:08:51",
  "op": [
    "delegate_vesting_shares",
    {
      "delegator": "steem",
      "delegatee": "calclavia",
      "vesting_shares": "1912.543513 VESTS"
    }
  ]
}

{
  "trx_id": "a9977412ac6a8abe15cf0acd66b698ef180d64fd",
  "block": 49185966,
  "trx_in_block": 0,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2020-12-05T11:05:48",
  "op": [
    "delegate_vesting_shares",
    {
      "delegator": "steem",
      "delegatee": "calclavia",
      "vesting_shares": "8540.282449 VESTS"
    }
  ]
}

{
  "trx_id": "c0973de678815780534629c63076c5f3d08ee559",
  "block": 48253665,
  "trx_in_block": 4,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2020-11-02T12:07:27",
  "op": [
    "delegate_vesting_shares",
    {
      "delegator": "steem",
      "delegatee": "calclavia",
      "vesting_shares": "1920.017158 VESTS"
    }
  ]
}

{
  "trx_id": "1fc0a910736bbc5a2445134a11de4ca91f7f92f0",
  "block": 43215081,
  "trx_in_block": 24,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2020-05-09T04:04:12",
  "op": [
    "delegate_vesting_shares",
    {
      "delegator": "steem",
      "delegatee": "calclavia",
      "vesting_shares": "8742.929023 VESTS"
    }
  ]
}

{
  "trx_id": "d227c6ec662a7cb7fd6520d83189ff5882c8212a",
  "block": 43190888,
  "trx_in_block": 21,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2020-05-08T07:25:36",
  "op": [
    "delegate_vesting_shares",
    {
      "delegator": "steem",
      "delegatee": "calclavia",
      "vesting_shares": "1953.311140 VESTS"
    }
  ]
}

{
  "trx_id": "e30acc416155fd1efc718a0993c74df84f12f5ac",
  "block": 42561335,
  "trx_in_block": 0,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2020-04-15T20:33:39",
  "op": [
    "delegate_vesting_shares",
    {
      "delegator": "steem",
      "delegatee": "calclavia",
      "vesting_shares": "8755.906442 VESTS"
    }
  ]
}

{
  "trx_id": "419e77adaa75cf3e5d1bcca4fb49be1378af24ab",
  "block": 40372561,
  "trx_in_block": 7,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2020-01-30T05:33:36",
  "op": [
    "comment",
    {
      "parent_author": "calclavia",
      "parent_permlink": "reinforcement-learning-using-asynchronous-advantage-actor-critic",
      "author": "steemitboard",
      "permlink": "steemitboard-notify-calclavia-20200130t053335000z",
      "title": "",
      "body": "Congratulations @calclavia! You received a personal award!\n\n<table><tr><td>https://steemitimages.com/70x70/http://steemitboard.com/@calclavia/birthday2.png</td><td>Happy Birthday! - You are on the Steem blockchain for 2 years!</td></tr></table>\n\n<sub>_You can view [your badges on your Steem Board](https://steemitboard.com/@calclavia) and compare to others on the [Steem Ranking](https://steemitboard.com/ranking/index.php?name=calclavia)_</sub>\n\n\n###### [Vote for @Steemitboard as a witness](https://v2.steemconnect.com/sign/account-witness-vote?witness=steemitboard&approve=1) to get one more award and increased upvotes!",
      "json_metadata": "{\"image\":[\"https://steemitboard.com/img/notify.png\"]}"
    }
  ]
}

{
  "trx_id": "ba7040585d8625ab29cd97770034de0716723b7c",
  "block": 35778792,
  "trx_in_block": 7,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2019-08-22T15:39:03",
  "op": [
    "transfer",
    {
      "from": "dtube",
      "to": "calclavia",
      "amount": "0.001 STEEM",
      "memo": "Time is running out, claim your DTube account now before anyone else can! Login at https://d.tube"
    }
  ]
}

{
  "trx_id": "925b8149fcdbdbab15806fcbd2d01aac257d8570",
  "block": 32844169,
  "trx_in_block": 4,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2019-05-12T13:48:18",
  "op": [
    "delegate_vesting_shares",
    {
      "delegator": "steem",
      "delegatee": "calclavia",
      "vesting_shares": "8951.529247 VESTS"
    }
  ]
}

{
  "trx_id": "d075eba6975568ff0e1acfbd9f21afc39572b9db",
  "block": 29899265,
  "trx_in_block": 5,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2019-01-30T04:46:06",
  "op": [
    "comment",
    {
      "parent_author": "calclavia",
      "parent_permlink": "reinforcement-learning-using-asynchronous-advantage-actor-critic",
      "author": "steemitboard",
      "permlink": "steemitboard-notify-calclavia-20190130t044606000z",
      "title": "",
      "body": "Congratulations @calclavia! You received a personal award!\n\n<table><tr><td>https://steemitimages.com/70x70/http://steemitboard.com/@calclavia/birthday1.png</td><td>Happy Birthday! - You are on the Steem blockchain for 1 year!</td></tr></table>\n\n<sub>_[Click here to view your Board](https://steemitboard.com/@calclavia)_</sub>\n\n\n> Support [SteemitBoard's project](https://steemit.com/@steemitboard)! **[Vote for its witness](https://v2.steemconnect.com/sign/account-witness-vote?witness=steemitboard&approve=1)** and **get one more award**!",
      "json_metadata": "{\"image\":[\"https://steemitboard.com/img/notify.png\"]}"
    }
  ]
}

{
  "trx_id": "004dc08b76aa69c88f10e61ba3d61ab67a630f4a",
  "block": 22489697,
  "trx_in_block": 31,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2018-05-16T20:09:48",
  "op": [
    "delegate_vesting_shares",
    {
      "delegator": "steem",
      "delegatee": "calclavia",
      "vesting_shares": "9151.081682 VESTS"
    }
  ]
}

{
  "trx_id": "0000000000000000000000000000000000000000",
  "block": 19617725,
  "trx_in_block": 4294967295,
  "op_in_trx": 0,
  "virtual_op": 9,
  "timestamp": "2018-02-06T00:19:15",
  "op": [
    "author_reward",
    {
      "author": "calclavia",
      "permlink": "reinforcement-learning-using-asynchronous-advantage-actor-critic",
      "sbd_payout": "0.012 SBD",
      "steem_payout": "0.000 STEEM",
      "vesting_payout": "6.136539 VESTS"
    }
  ]
}

{
  "trx_id": "fd97a8d0299d20c4a1a1317522c0a6a034059e62",
  "block": 19446871,
  "trx_in_block": 29,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2018-01-31T01:43:15",
  "op": [
    "vote",
    {
      "voter": "calclavia",
      "author": "traplord",
      "permlink": "the-trouble-with-virtual-teams",
      "weight": 10000
    }
  ]
}

{
  "trx_id": "bbb65d0b71edcffa58a3a03255113eabaa1ac503",
  "block": 19446850,
  "trx_in_block": 1,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2018-01-31T01:42:12",
  "op": [
    "vote",
    {
      "voter": "calclavia",
      "author": "traplord",
      "permlink": "the-importance-of-steemit-and-the-falling-price-of-bitcoin",
      "weight": 10000
    }
  ]
}

{
  "trx_id": "82e619d85019ee5f0399183700f1ad7c05430609",
  "block": 19427872,
  "trx_in_block": 35,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2018-01-30T09:52:03",
  "op": [
    "vote",
    {
      "voter": "konchozzz",
      "author": "calclavia",
      "permlink": "reinforcement-learning-using-asynchronous-advantage-actor-critic",
      "weight": 10000
    }
  ]
}

{
  "trx_id": "09695b4f9a2278d3742bbb821f0b05e63918cb88",
  "block": 19416718,
  "trx_in_block": 13,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2018-01-30T00:33:36",
  "op": [
    "custom_json",
    {
      "required_auths": [],
      "required_posting_auths": [
        "calclavia"
      ],
      "id": "follow",
      "json": "[\"follow\",{\"follower\":\"calclavia\",\"following\":\"dtube\",\"what\":[\"blog\"]}]"
    }
  ]
}

{
  "trx_id": "ab1644a1290abf4d20a0444c8136efae14bc7a9b",
  "block": 19416716,
  "trx_in_block": 9,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2018-01-30T00:33:30",
  "op": [
    "account_update",
    {
      "account": "calclavia",
      "posting": {
        "weight_threshold": 1,
        "account_auths": [
          [
            "dtube.app",
            1
          ]
        ],
        "key_auths": [
          [
            "STM64Bx4AWNRF96QAx1FHDzYHn2o7MpfcExDyRrW3dwYAux77gLmf",
            1
          ]
        ]
      },
      "memo_key": "STM6g9QdH4CCVs1AReQLZZSraFbajet76f9TEPGY6Ho4RZ52CZYP3",
      "json_metadata": "{\"profile\":{\"profile_image\":\"https://cdn-images-1.medium.com/fit/c/100/100/0*m39fO6oY3Kqz66yX.\",\"name\":\"Henry\",\"about\":\"Entrepreneur, Artificial Intelligence Researcher. Founder of Altum Inc and Calclavia.\",\"location\":\"California\",\"website\":\"https://calclavia.com\"}}"
    }
  ]
}

{
  "trx_id": "a714d4ae44a327df60ad033553ec51cb31a17f4e",
  "block": 19416670,
  "trx_in_block": 1,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2018-01-30T00:31:12",
  "op": [
    "vote",
    {
      "voter": "calclavia",
      "author": "calclavia",
      "permlink": "reinforcement-learning-using-asynchronous-advantage-actor-critic",
      "weight": 10000
    }
  ]
}

{
  "trx_id": "5d643ebb6f5eb5f4d53fcb4d18a0ae82c84dd393",
  "block": 19416660,
  "trx_in_block": 2,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2018-01-30T00:30:42",
  "op": [
    "vote",
    {
      "voter": "calclavia",
      "author": "midasexpo",
      "permlink": "my-best-deep-dream-creations-so-far",
      "weight": 10000
    }
  ]
}

{
  "trx_id": "615b591bc2de22b1042fcc88c470dab5b9a14e65",
  "block": 19416552,
  "trx_in_block": 25,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2018-01-30T00:25:18",
  "op": [
    "comment",
    {
      "parent_author": "",
      "parent_permlink": "a3c",
      "author": "calclavia",
      "permlink": "reinforcement-learning-using-asynchronous-advantage-actor-critic",
      "title": "Reinforcement Learning using Asynchronous Advantage Actor Critic",
      "body": "Reinforcement learning is an extremely exciting field that has pushed the boundaries of artificial intelligence. In my research, I stumbled upon an effective reinforcement learning method called Asynchronous Advantage Actor Critic (A3C) published by DeepMind. This algorithm beats the famous DQN by quite a margin and also seems to yield more stable results. I wanted to give a high level explanation in this post of how the algorithm works, hopefully inspiring more people to apply it in their projects. If you’re interested in the code, I implemented the [algorithm](https://github.com/calclavia/rl) using Tensorflow and Keras inspired by this [Medium article](https://medium.com/emergent-future/simple-reinforcement-learning-with-tensorflow-part-8-asynchronous-actor-critic-agents-a3c-c88f72a5e9f2#.dgiztjv7l). The library is compatible with [OpenAI’s Gym API](https://gym.openai.com/).\n\n# Actor Critic Models\nBefore we dive into the asynchronous part, I’d like to explain Actor-Critic (AC) learning models. In a reinforcement learning problem, an agent exists in some state _s_ and tries to choose an action _a_ to maximize its discounted future rewards.\n\n<center>\n![Reinforcement Learning Diagram](https://cdn-images-1.medium.com/max/800/0*Rda_s6qNiZhxmQEJ.png)\n</center>\n\nThe AC agent is comprised of an actor and a critic. The actor attempts to learn a policy _π(s)_ (AKA the rule that the agent follows) by receiving feedback from a critic. The critic learns a value function _V(s)_ (the expected return in rewards), which is used to determine how advantageous it is to be in a particular state. The advantage is defined as _A(s) = Q(s, a) - V(s)_. In practice, we don’t want to compute _Q(s, a)_. Instead, we formulate an estimate of the advantage function as _A(s) = r + γV(s’) - V(s)_, where _r_ is the current reward and _γ_ is the discount factor. This achieves the same result without needing to learn the _Q_ function. An even more effective method would be to use [generalized advantage estimation](https://arxiv.org/pdf/1506.02438.pdf).\n\n## Objective Functions\nLooking at the actor-critic agent from a neural network perspective, we would give the agent two outputs: value and policy. The value output predicts a scalar that learns the value function _V(s)_. The policy output _π(s)_ (softmax activation) is a vector that represents a probability distribution over the actions. We pick the action non-deterministically by sampling from this probability distribution. We denote _π(a | s)_ as the probability of the sampled action a given state _s_.\n\nWe arrive at the following loss functions (we want to minimize these). _R_ represents the discounted future reward (_R = r + γV(s’)_).\n\n> Value Loss: _L = Σ(R - V(s))²_ (Sum Squared Error)\n> Policy Loss: _L = -log(π(a | s)) * A(s)_\n\nBut not so fast! While the loss functions above would work, it is better to introduce the entropy _H(π)_ to the equation.\n\n> _H(π) = - Σ(P(x) log(P(x))_\n\nEntropy is a measure of how spread out the probabilities are. The higher the entropy, the more similar each action’s probability will be, which makes the agent more uncertain about which action to choose. Entropy can be added to the loss function to encourage exploration by preventing the agent from being too decisive and converging at local optima\n\n> Policy Loss: _L = - log(π(a | s)) * A(s) - β*H(π)_\n\nWhen we combine the two loss functions, we get the loss function for the model overall:\n\n> _L = 0.5 * Σ(R — V(s))² - log(π(a | s)) * A(s) - β*H(π)_\n\nNotice that the loss for value is set to 50% to make policy learning faster than value learning. For more information on the derivations of these loss functions, I recommend watching [David Silver’s RL lecture videos](https://www.youtube.com/watch?v=KHZVXao4qXs). With that, we can train our AC agent!\n\n# Asynchronous\nThe interesting part about A3C is the first A — asynchronous. DeepMind’s paper showed that by introducing asynchronous training, we can reduce the correlation between episodes, improving various methods of learning including Q-learning (better data efficiency). It is also a more efficient use of multi-core CPUs, allowing us to train agents to do quite amazing things with just a laptop.\n\nhttps://cdn-images-1.medium.com/max/800/1*YtnGhtSAMnnHSL8PvS7t_w.png\n\nA3C works by spawning minion AC agents, each performing actions in their own separate environments and updating the master neural network after a certain amount of actions have been taken. The individual agents sync their weights with the master network after every gradient update.\n\nHowever, [more recent research from OpenAI](https://blog.openai.com/baselines-acktr-a2c/) suggests that A2C (without asynchronous learning) performs equally well when using GPUs. We can argue that the key benefit of A3C is that there are parallel agents learning at the same time, allowing a policy to be evaluated on multiple trajectories simultaneously.\n\nThat’s it for a high level overview of A3C. If you’re interested in checking out a detailed implementation of the algorithm, be sure to check out my [Github repository](https://github.com/calclavia/rl).\n\nI’ll be following up on this post shortly on how I applied A3C to a mobile game I developed called _Relay_ . Feel free to leave me suggestions or ask questions in the comments section!",
      "json_metadata": "{\"tags\":[\"ai\",\"deep\",\"reinforcement\",\"learning\",\"a3c\"],\"image\":[\"https://cdn-images-1.medium.com/max/800/0*Rda_s6qNiZhxmQEJ.png\",\"https://cdn-images-1.medium.com/max/800/1*YtnGhtSAMnnHSL8PvS7t_w.png\"],\"links\":[\"https://github.com/calclavia/rl\",\"https://medium.com/emergent-future/simple-reinforcement-learning-with-tensorflow-part-8-asynchronous-actor-critic-agents-a3c-c88f72a5e9f2#.dgiztjv7l\",\"https://gym.openai.com/\",\"https://arxiv.org/pdf/1506.02438.pdf\",\"https://www.youtube.com/watch?v=KHZVXao4qXs\",\"https://blog.openai.com/baselines-acktr-a2c/\"],\"app\":\"steemit/0.1\",\"format\":\"markdown\"}"
    }
  ]
}

{
  "trx_id": "d345fb5b8b99202bce2de759a5ba565831415e67",
  "block": 19416545,
  "trx_in_block": 9,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2018-01-30T00:24:57",
  "op": [
    "comment",
    {
      "parent_author": "",
      "parent_permlink": "a3c",
      "author": "calclavia",
      "permlink": "reinforcement-learning-using-asynchronous-advantage-actor-critic",
      "title": "Reinforcement Learning using Asynchronous Advantage Actor Critic",
      "body": "Reinforcement learning is an extremely exciting field that has pushed the boundaries of artificial intelligence. In my research, I stumbled upon an effective reinforcement learning method called Asynchronous Advantage Actor Critic (A3C) published by DeepMind. This algorithm beats the famous DQN by quite a margin and also seems to yield more stable results. I wanted to give a high level explanation in this post of how the algorithm works, hopefully inspiring more people to apply it in their projects. If you’re interested in the code, I implemented the [algorithm](https://github.com/calclavia/rl) using Tensorflow and Keras inspired by this [Medium article](https://medium.com/emergent-future/simple-reinforcement-learning-with-tensorflow-part-8-asynchronous-actor-critic-agents-a3c-c88f72a5e9f2#.dgiztjv7l). The library is compatible with [OpenAI’s Gym API](https://gym.openai.com/).\n\n# Actor Critic Models\nBefore we dive into the asynchronous part, I’d like to explain Actor-Critic (AC) learning models. In a reinforcement learning problem, an agent exists in some state _s_ and tries to choose an action _a_ to maximize its discounted future rewards.\n\n<center>\n![Reinforcement Learning Diagram](https://cdn-images-1.medium.com/max/800/0*Rda_s6qNiZhxmQEJ.png)\n</center>\n\nThe AC agent is comprised of an actor and a critic. The actor attempts to learn a policy _π(s)_ (AKA the rule that the agent follows) by receiving feedback from a critic. The critic learns a value function _V(s)_ (the expected return in rewards), which is used to determine how advantageous it is to be in a particular state. The advantage is defined as _A(s) = Q(s, a) - V(s)_. In practice, we don’t want to compute _Q(s, a)_. Instead, we formulate an estimate of the advantage function as _A(s) = r + γV(s’) - V(s)_, where _r_ is the current reward and _γ_ is the discount factor. This achieves the same result without needing to learn the _Q_ function. An even more effective method would be to use [generalized advantage estimation](https://arxiv.org/pdf/1506.02438.pdf).\n\n## Objective Functions\nLooking at the actor-critic agent from a neural network perspective, we would give the agent two outputs: value and policy. The value output predicts a scalar that learns the value function _V(s)_. The policy output _π(s)_ (softmax activation) is a vector that represents a probability distribution over the actions. We pick the action non-deterministically by sampling from this probability distribution. We denote _π(a | s)_ as the probability of the sampled action a given state _s_.\n\nWe arrive at the following loss functions (we want to minimize these). _R_ represents the discounted future reward (_R = r + γV(s’)_).\n\n> Value Loss: _L = Σ(R - V(s))²_ (Sum Squared Error)\n> Policy Loss: _L = -log(π(a | s)) * A(s)_\n\nBut not so fast! While the loss functions above would work, it is better to introduce the entropy _H(π)_ to the equation.\n\n> _H(π) = - Σ(P(x) log(P(x))_\n\nEntropy is a measure of how spread out the probabilities are. The higher the entropy, the more similar each action’s probability will be, which makes the agent more uncertain about which action to choose. Entropy can be added to the loss function to encourage exploration by preventing the agent from being too decisive and converging at local optima\n\n> Policy Loss: _L = - log(π(a | s)) * A(s) - β*H(π)_\n\nWhen we combine the two loss functions, we get the loss function for the model overall:\n\n> _L = 0.5 * Σ(R — V(s))² - log(π(a | s)) * A(s) - β*H(π)_\n\nNotice that the loss for value is set to 50% to make policy learning faster than value learning. For more information on the derivations of these loss functions, I recommend watching [David Silver’s RL lecture videos](https://www.youtube.com/watch?v=KHZVXao4qXs). With that, we can train our AC agent!\n\n# Asynchronous\nThe interesting part about A3C is the first A — asynchronous. DeepMind’s paper showed that by introducing asynchronous training, we can reduce the correlation between episodes, improving various methods of learning including Q-learning (better data efficiency). It is also a more efficient use of multi-core CPUs, allowing us to train agents to do quite amazing things with just a laptop.\n\nhttps://cdn-images-1.medium.com/max/800/1*YtnGhtSAMnnHSL8PvS7t_w.png\n\nA3C works by spawning minion AC agents, each performing actions in their own separate environments and updating the master neural network after a certain amount of actions have been taken. The individual agents sync their weights with the master network after every gradient update.\n\nHowever, [more recent research from OpenAI](https://blog.openai.com/baselines-acktr-a2c/) suggests that A2C (without asynchronous learning) performs equally well when using GPUs. We can argue that the key benefit of A3C is that there are parallel agents learning at the same time, allowing a policy to be evaluated on multiple trajectories simultaneously.\n\nThat’s it for a high level overview of A3C. If you’re interested in checking out a detailed implementation of the algorithm, be sure to check out my [Github repository](https://github.com/calclavia/rl).\n\nI’ll be following up on this post shortly on how I applied A3C to a mobile game I developed called _Relay_ . Feel free to leave me suggestions or ask questions in the comments section!",
      "json_metadata": "{\"tags\":[\"ai\",\"deep\",\"reinforcement\",\"learning\",\"a3c\"],\"image\":[\"https://cdn-images-1.medium.com/max/800/0*Rda_s6qNiZhxmQEJ.png\",\"https://cdn-images-1.medium.com/max/800/1*YtnGhtSAMnnHSL8PvS7t_w.png\"],\"links\":[\"https://github.com/calclavia/rl\",\"https://medium.com/emergent-future/simple-reinforcement-learning-with-tensorflow-part-8-asynchronous-actor-critic-agents-a3c-c88f72a5e9f2#.dgiztjv7l\",\"https://gym.openai.com/\",\"https://arxiv.org/pdf/1506.02438.pdf\",\"https://www.youtube.com/watch?v=KHZVXao4qXs\",\"https://blog.openai.com/baselines-acktr-a2c/\"],\"app\":\"steemit/0.1\",\"format\":\"markdown\"}"
    }
  ]
}

{
  "trx_id": "81c2b4875170e31c553062ea7be806ec707b10ab",
  "block": 19416537,
  "trx_in_block": 17,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2018-01-30T00:24:33",
  "op": [
    "vote",
    {
      "voter": "traplord",
      "author": "calclavia",
      "permlink": "reinforcement-learning-using-asynchronous-advantage-actor-critic",
      "weight": 10000
    }
  ]
}

{
  "trx_id": "fca0fcb50c522df7c410a3f310c5e9e1371f1ea6",
  "block": 19416530,
  "trx_in_block": 33,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2018-01-30T00:24:12",
  "op": [
    "vote",
    {
      "voter": "calclavia",
      "author": "traplord",
      "permlink": "introductory-post",
      "weight": 10000
    }
  ]
}

{
  "trx_id": "265054c97caa088cf91b5c9eb357a1d1bf6eabf4",
  "block": 19416519,
  "trx_in_block": 51,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2018-01-30T00:23:39",
  "op": [
    "account_update",
    {
      "account": "calclavia",
      "memo_key": "STM6g9QdH4CCVs1AReQLZZSraFbajet76f9TEPGY6Ho4RZ52CZYP3",
      "json_metadata": "{\"profile\":{\"profile_image\":\"https://cdn-images-1.medium.com/fit/c/100/100/0*m39fO6oY3Kqz66yX.\",\"name\":\"Henry\",\"about\":\"Entrepreneur, Artificial Intelligence Researcher. Founder of Altum Inc and Calclavia.\",\"location\":\"California\",\"website\":\"https://calclavia.com\"}}"
    }
  ]
}

{
  "trx_id": "33db005546e4d17af9b2eabc1f3b8dc9e4beb4c5",
  "block": 19416470,
  "trx_in_block": 19,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2018-01-30T00:21:12",
  "op": [
    "comment",
    {
      "parent_author": "",
      "parent_permlink": "a3c",
      "author": "calclavia",
      "permlink": "reinforcement-learning-using-asynchronous-advantage-actor-critic",
      "title": "Reinforcement Learning using Asynchronous Advantage Actor Critic",
      "body": "@@ -151,16 +151,30 @@\n fective \n+reinforcement \n learning\n",
      "json_metadata": "{\"tags\":[\"a3c\",\"deep\",\"reinforcement\",\"learning\",\"algorithm\"],\"image\":[\"https://cdn-images-1.medium.com/max/800/0*Rda_s6qNiZhxmQEJ.png\",\"https://cdn-images-1.medium.com/max/800/1*YtnGhtSAMnnHSL8PvS7t_w.png\"],\"links\":[\"https://github.com/calclavia/rl\",\"https://medium.com/emergent-future/simple-reinforcement-learning-with-tensorflow-part-8-asynchronous-actor-critic-agents-a3c-c88f72a5e9f2#.dgiztjv7l\",\"https://gym.openai.com/\",\"https://arxiv.org/pdf/1506.02438.pdf\",\"https://www.youtube.com/watch?v=KHZVXao4qXs\",\"https://blog.openai.com/baselines-acktr-a2c/\"],\"app\":\"steemit/0.1\",\"format\":\"markdown\"}"
    }
  ]
}

{
  "trx_id": "2939757efc851e5b834c8480869c2a0b9e6d2591",
  "block": 19416453,
  "trx_in_block": 4,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2018-01-30T00:20:21",
  "op": [
    "comment",
    {
      "parent_author": "",
      "parent_permlink": "a3c",
      "author": "calclavia",
      "permlink": "reinforcement-learning-using-asynchronous-advantage-actor-critic",
      "title": "Reinforcement Learning using Asynchronous Advantage Actor Critic",
      "body": "@@ -4195,16 +4195,86 @@\n aptop.%0A%0A\n+https://cdn-images-1.medium.com/max/800/1*YtnGhtSAMnnHSL8PvS7t_w.png%0A%0A\n A3C work\n",
      "json_metadata": "{\"tags\":[\"a3c\",\"deep\",\"reinforcement\",\"learning\",\"algorithm\"],\"image\":[\"https://cdn-images-1.medium.com/max/800/0*Rda_s6qNiZhxmQEJ.png\",\"https://cdn-images-1.medium.com/max/800/1*YtnGhtSAMnnHSL8PvS7t_w.png\"],\"links\":[\"https://github.com/calclavia/rl\",\"https://medium.com/emergent-future/simple-reinforcement-learning-with-tensorflow-part-8-asynchronous-actor-critic-agents-a3c-c88f72a5e9f2#.dgiztjv7l\",\"https://gym.openai.com/\",\"https://arxiv.org/pdf/1506.02438.pdf\",\"https://www.youtube.com/watch?v=KHZVXao4qXs\",\"https://blog.openai.com/baselines-acktr-a2c/\"],\"app\":\"steemit/0.1\",\"format\":\"markdown\"}"
    }
  ]
}

{
  "trx_id": "221fb273c9576dfe229528bb71ac69f81d9937b8",
  "block": 19416437,
  "trx_in_block": 40,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2018-01-30T00:19:33",
  "op": [
    "comment",
    {
      "parent_author": "calclavia",
      "parent_permlink": "reinforcement-learning-using-asynchronous-advantage-actor-critic",
      "author": "cheetah",
      "permlink": "cheetah-re-calclaviareinforcement-learning-using-asynchronous-advantage-actor-critic",
      "title": "",
      "body": "Hi! I am a robot. I just upvoted you! I found similar content that readers might be interested in:\nhttps://medium.com/@henrymao/reinforcement-learning-using-asynchronous-advantage-actor-critic-704147f91686",
      "json_metadata": ""
    }
  ]
}

{
  "trx_id": "815cd54349bd4651bb4c949b87a5b9ab6f0465e8",
  "block": 19416436,
  "trx_in_block": 9,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2018-01-30T00:19:30",
  "op": [
    "vote",
    {
      "voter": "cheetah",
      "author": "calclavia",
      "permlink": "reinforcement-learning-using-asynchronous-advantage-actor-critic",
      "weight": 8
    }
  ]
}

{
  "trx_id": "ce5304208a8cfb6b51963917056ab75856c9df40",
  "block": 19416431,
  "trx_in_block": 51,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2018-01-30T00:19:15",
  "op": [
    "comment",
    {
      "parent_author": "",
      "parent_permlink": "a3c",
      "author": "calclavia",
      "permlink": "reinforcement-learning-using-asynchronous-advantage-actor-critic",
      "title": "Reinforcement Learning using Asynchronous Advantage Actor Critic",
      "body": "Reinforcement learning is an extremely exciting field that has pushed the boundaries of artificial intelligence. In my research, I stumbled upon an effective learning method called Asynchronous Advantage Actor Critic (A3C) published by DeepMind. This algorithm beats the famous DQN by quite a margin and also seems to yield more stable results. I wanted to give a high level explanation in this post of how the algorithm works, hopefully inspiring more people to apply it in their projects. If you’re interested in the code, I implemented the [algorithm](https://github.com/calclavia/rl) using Tensorflow and Keras inspired by this [Medium article](https://medium.com/emergent-future/simple-reinforcement-learning-with-tensorflow-part-8-asynchronous-actor-critic-agents-a3c-c88f72a5e9f2#.dgiztjv7l). The library is compatible with [OpenAI’s Gym API](https://gym.openai.com/).\n\n# Actor Critic Models\nBefore we dive into the asynchronous part, I’d like to explain Actor-Critic (AC) learning models. In a reinforcement learning problem, an agent exists in some state _s_ and tries to choose an action _a_ to maximize its discounted future rewards.\n\n<center>\n![Reinforcement Learning Diagram](https://cdn-images-1.medium.com/max/800/0*Rda_s6qNiZhxmQEJ.png)\n</center>\n\nThe AC agent is comprised of an actor and a critic. The actor attempts to learn a policy _π(s)_ (AKA the rule that the agent follows) by receiving feedback from a critic. The critic learns a value function _V(s)_ (the expected return in rewards), which is used to determine how advantageous it is to be in a particular state. The advantage is defined as _A(s) = Q(s, a) - V(s)_. In practice, we don’t want to compute _Q(s, a)_. Instead, we formulate an estimate of the advantage function as _A(s) = r + γV(s’) - V(s)_, where _r_ is the current reward and _γ_ is the discount factor. This achieves the same result without needing to learn the _Q_ function. An even more effective method would be to use [generalized advantage estimation](https://arxiv.org/pdf/1506.02438.pdf).\n\n## Objective Functions\nLooking at the actor-critic agent from a neural network perspective, we would give the agent two outputs: value and policy. The value output predicts a scalar that learns the value function _V(s)_. The policy output _π(s)_ (softmax activation) is a vector that represents a probability distribution over the actions. We pick the action non-deterministically by sampling from this probability distribution. We denote _π(a | s)_ as the probability of the sampled action a given state _s_.\n\nWe arrive at the following loss functions (we want to minimize these). _R_ represents the discounted future reward (_R = r + γV(s’)_).\n\n> Value Loss: _L = Σ(R - V(s))²_ (Sum Squared Error)\n> Policy Loss: _L = -log(π(a | s)) * A(s)_\n\nBut not so fast! While the loss functions above would work, it is better to introduce the entropy _H(π)_ to the equation.\n\n> _H(π) = - Σ(P(x) log(P(x))_\n\nEntropy is a measure of how spread out the probabilities are. The higher the entropy, the more similar each action’s probability will be, which makes the agent more uncertain about which action to choose. Entropy can be added to the loss function to encourage exploration by preventing the agent from being too decisive and converging at local optima\n\n> Policy Loss: _L = - log(π(a | s)) * A(s) - β*H(π)_\n\nWhen we combine the two loss functions, we get the loss function for the model overall:\n\n> _L = 0.5 * Σ(R — V(s))² - log(π(a | s)) * A(s) - β*H(π)_\n\nNotice that the loss for value is set to 50% to make policy learning faster than value learning. For more information on the derivations of these loss functions, I recommend watching [David Silver’s RL lecture videos](https://www.youtube.com/watch?v=KHZVXao4qXs). With that, we can train our AC agent!\n\n# Asynchronous\nThe interesting part about A3C is the first A — asynchronous. DeepMind’s paper showed that by introducing asynchronous training, we can reduce the correlation between episodes, improving various methods of learning including Q-learning (better data efficiency). It is also a more efficient use of multi-core CPUs, allowing us to train agents to do quite amazing things with just a laptop.\n\nA3C works by spawning minion AC agents, each performing actions in their own separate environments and updating the master neural network after a certain amount of actions have been taken. The individual agents sync their weights with the master network after every gradient update.\n\nHowever, [more recent research from OpenAI](https://blog.openai.com/baselines-acktr-a2c/) suggests that A2C (without asynchronous learning) performs equally well when using GPUs. We can argue that the key benefit of A3C is that there are parallel agents learning at the same time, allowing a policy to be evaluated on multiple trajectories simultaneously.\n\nThat’s it for a high level overview of A3C. If you’re interested in checking out a detailed implementation of the algorithm, be sure to check out my [Github repository](https://github.com/calclavia/rl).\n\nI’ll be following up on this post shortly on how I applied A3C to a mobile game I developed called _Relay_ . Feel free to leave me suggestions or ask questions in the comments section!",
      "json_metadata": "{\"tags\":[\"a3c\",\"deep\",\"reinforcement\",\"learning\",\"algorithm\"],\"image\":[\"https://cdn-images-1.medium.com/max/800/0*Rda_s6qNiZhxmQEJ.png\"],\"links\":[\"https://github.com/calclavia/rl\",\"https://medium.com/emergent-future/simple-reinforcement-learning-with-tensorflow-part-8-asynchronous-actor-critic-agents-a3c-c88f72a5e9f2#.dgiztjv7l\",\"https://gym.openai.com/\",\"https://arxiv.org/pdf/1506.02438.pdf\",\"https://www.youtube.com/watch?v=KHZVXao4qXs\",\"https://blog.openai.com/baselines-acktr-a2c/\"],\"app\":\"steemit/0.1\",\"format\":\"markdown\"}"
    }
  ]
}

{
  "trx_id": "23223b433aed9f9ddceb76b5838c2eec990eda82",
  "block": 19416299,
  "trx_in_block": 41,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2018-01-30T00:12:39",
  "op": [
    "vote",
    {
      "voter": "traplord",
      "author": "calclavia",
      "permlink": "re-traplord-introductory-post-20180129t235143776z",
      "weight": 10000
    }
  ]
}

{
  "trx_id": "83635a66a30eda7ffaef492bf3102e982ace75d8",
  "block": 19415954,
  "trx_in_block": 2,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2018-01-29T23:55:24",
  "op": [
    "account_update",
    {
      "account": "calclavia",
      "memo_key": "STM6g9QdH4CCVs1AReQLZZSraFbajet76f9TEPGY6Ho4RZ52CZYP3",
      "json_metadata": "{\"profile\":{\"profile_image\":\"https://cdn-images-1.medium.com/fit/c/100/100/0*m39fO6oY3Kqz66yX.\",\"name\":\"calclavia\",\"about\":\"Entrepreneur, Software Engineer. Founder of Altum Inc and Calclavia. Pursuing machine learning and AI research.\",\"location\":\"California\",\"website\":\"https://calclavia.com\"}}"
    }
  ]
}

{
  "trx_id": "284f06e86f86e1d955f022239276b103d2278886",
  "block": 19415916,
  "trx_in_block": 24,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2018-01-29T23:53:30",
  "op": [
    "account_update",
    {
      "account": "calclavia",
      "memo_key": "STM6g9QdH4CCVs1AReQLZZSraFbajet76f9TEPGY6Ho4RZ52CZYP3",
      "json_metadata": "{\"profile\":{\"profile_image\":\"https://cdn-images-1.medium.com/fit/c/100/100/0*m39fO6oY3Kqz66yX.\"}}"
    }
  ]
}

{
  "trx_id": "b1a258e63041261f707a2236d4ef3249c42ac677",
  "block": 19415881,
  "trx_in_block": 56,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2018-01-29T23:51:45",
  "op": [
    "comment",
    {
      "parent_author": "traplord",
      "parent_permlink": "introductory-post",
      "author": "calclavia",
      "permlink": "re-traplord-introductory-post-20180129t235143776z",
      "title": "",
      "body": "Glad you made it on the platform!",
      "json_metadata": "{\"tags\":[\"introduceyourself\"],\"app\":\"steemit/0.1\"}"
    }
  ]
}

{
  "trx_id": "04a455b720a3263536fe2a182f8a537e4212e468",
  "block": 19415872,
  "trx_in_block": 12,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2018-01-29T23:51:18",
  "op": [
    "custom_json",
    {
      "required_auths": [],
      "required_posting_auths": [
        "calclavia"
      ],
      "id": "follow",
      "json": "[\"follow\",{\"follower\":\"calclavia\",\"following\":\"traplord\",\"what\":[\"blog\"]}]"
    }
  ]
}

{
  "trx_id": "a7feebfd8c0f65193e891451143f612ddeb7a39b",
  "block": 19415854,
  "trx_in_block": 23,
  "op_in_trx": 0,
  "virtual_op": 0,
  "timestamp": "2018-01-29T23:50:21",
  "op": [
    "account_create_with_delegation",
    {
      "fee": "0.500 STEEM",
      "delegation": "29700.000000 VESTS",
      "creator": "steem",
      "new_account_name": "calclavia",
      "owner": {
        "weight_threshold": 1,
        "account_auths": [],
        "key_auths": [
          [
            "STM6Hb8SSQNGQDTdQBryix1PsqGGwwz7c8uD1bx1JA2Xt34qD6BkX",
            1
          ]
        ]
      },
      "active": {
        "weight_threshold": 1,
        "account_auths": [],
        "key_auths": [
          [
            "STM6xG22tmKi3TeEc8djYr8zTCjfaP62iMzNkdJU4ke9sMFKu2A7K",
            1
          ]
        ]
      },
      "posting": {
        "weight_threshold": 1,
        "account_auths": [],
        "key_auths": [
          [
            "STM64Bx4AWNRF96QAx1FHDzYHn2o7MpfcExDyRrW3dwYAux77gLmf",
            1
          ]
        ]
      },
      "memo_key": "STM6g9QdH4CCVs1AReQLZZSraFbajet76f9TEPGY6Ho4RZ52CZYP3",
      "json_metadata": "",
      "extensions": []
    }
  ]
}